goroutine 2397 [running]: runtime/pprof.writeGoroutineStacks({0x631b840, 0xc042fa1e40}) GOROOT/src/runtime/pprof/pprof.go:693 +0x70 runtime/pprof.writeGoroutine({0x631b840, 0xc042fa1e40}, 0xc064c39ae8) GOROOT/src/runtime/pprof/pprof.go:682 +0x2b runtime/pprof.(*Profile).WriteTo(0x4ee79f2, {0x631b840, 0xc042fa1e40}, 0x4edccb7) GOROOT/src/runtime/pprof/pprof.go:331 +0x14b github.com/cockroachdb/cockroach/pkg/server/goroutinedumper.takeGoroutineDump({0xc059e81030, 0xd}) github.com/cockroachdb/cockroach/pkg/server/goroutinedumper/goroutinedumper.go:170 +0x109 github.com/cockroachdb/cockroach/pkg/server/goroutinedumper.(*GoroutineDumper).MaybeDump(0xc00a8f8aa0, {0x63c85b8, 0xc018d32db0}, 0xc099c68800, 0xa9a) github.com/cockroachdb/cockroach/pkg/server/goroutinedumper/goroutinedumper.go:101 +0x2b6 github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1({0x63c85b8, 0xc018d32db0}) github.com/cockroachdb/cockroach/pkg/server/env_sampler.go:181 +0x46b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 1 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/cli.waitForShutdown(0xc0009a0060, 0xc000d3ab40, 0xc001495800, 0xc000a56fc0, 0xc000cbc2f0) github.com/cockroachdb/cockroach/pkg/cli/start.go:727 +0x1c5 github.com/cockroachdb/cockroach/pkg/cli.runStart(0x91c4e20, {0xc000549ad0, 0x46746e, 0xc0009674d8}, 0x0) github.com/cockroachdb/cockroach/pkg/cli/start.go:674 +0x82a github.com/cockroachdb/cockroach/pkg/cli.runStartJoin(0x5, {0xc000884580, 0x20, 0x200}) github.com/cockroachdb/cockroach/pkg/cli/start.go:343 +0x25 github.com/cockroachdb/cockroach/pkg/cli/clierrorplus.MaybeDecorateError.func1(0xc0010c6f00, {0xc000884580, 0xc000549cc0, 0xc000549c98}) github.com/cockroachdb/cockroach/pkg/cli/clierrorplus/decorate_error.go:67 +0x43 github.com/cockroachdb/cockroach/pkg/cli/clierrorplus.MaybeShoutError.func1(0x91c4e20, {0xc000884580, 0x0, 0x8}) github.com/cockroachdb/cockroach/pkg/cli/clierrorplus/shout.go:24 +0x22 github.com/spf13/cobra.(*Command).execute(0x91c4e20, {0xc000884380, 0x8, 0x8}) github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:856 +0x60e github.com/spf13/cobra.(*Command).ExecuteC(0x91c3f20) github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:974 +0x3bc github.com/spf13/cobra.(*Command).Execute(...) github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:902 github.com/cockroachdb/cockroach/pkg/cli.Run(...) github.com/cockroachdb/cockroach/pkg/cli/cli.go:298 github.com/cockroachdb/cockroach/pkg/cli.doMain(0x91c4e20, {0xc000c3e81a, 0x5}) github.com/cockroachdb/cockroach/pkg/cli/cli.go:140 +0x1e9 github.com/cockroachdb/cockroach/pkg/cli.Main() github.com/cockroachdb/cockroach/pkg/cli/cli.go:65 +0x149 main.main() main/pkg/cmd/cockroach/main.go:26 +0x17 goroutine 19 [chan receive]: github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon() github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x5b created by github.com/cockroachdb/cockroach/pkg/util/log.init.5 github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x25 goroutine 20 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher() github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0xe5 created by github.com/cockroachdb/cockroach/pkg/util/log.init.5 github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x33 goroutine 50 [syscall, 716 minutes]: os/signal.signal_recv() GOROOT/src/runtime/sigqueue.go:169 +0x98 os/signal.loop() GOROOT/src/os/signal/signal_unix.go:24 +0x19 created by os/signal.Notify.func1.1 GOROOT/src/os/signal/signal.go:151 +0x2c goroutine 25 [chan receive]: github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1() github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 +0x127 created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0 github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:157 +0x25 goroutine 28 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup.func1() github.com/cockroachdb/cockroach/pkg/cli/cli.go:330 +0x9d created by github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup github.com/cockroachdb/cockroach/pkg/cli/cli.go:328 +0x112 goroutine 14 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0009a7e00) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 15 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc0009a7ec0) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 16 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36300) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 66 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d363c0) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 67 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36480) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 68 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36540) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 69 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36600) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 70 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d366c0) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 71 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36780) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 72 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36840) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 73 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36900) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 74 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d369c0) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 75 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36a80) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 76 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36b40) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 77 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36c00) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 78 [chan receive, 716 minutes]: github.com/klauspost/compress/zstd.(*blockDec).startDecoder(0xc000d36cc0) github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:211 +0x94 created by github.com/klauspost/compress/zstd.newBlockDec github.com/klauspost/compress/zstd/external/com_github_klauspost_compress/zstd/blockdec.go:117 +0x187 goroutine 79 [select]: go.opencensus.io/stats/view.(*worker).start(0xc000884600) go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:276 +0xb9 created by go.opencensus.io/stats/view.init.0 go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:34 +0x92 goroutine 56 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup.func2() github.com/cockroachdb/cockroach/pkg/cli/cli.go:352 +0xab created by github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup github.com/cockroachdb/cockroach/pkg/cli/cli.go:350 +0x1f6 goroutine 57 [chan receive, 716 minutes]: github.com/getsentry/sentry-go.(*HTTPTransport).worker(0xc0001f2000) github.com/getsentry/sentry-go/external/com_github_getsentry_sentry_go/transport.go:387 +0x129 created by github.com/getsentry/sentry-go.(*HTTPTransport).Configure.func1 github.com/getsentry/sentry-go/external/com_github_getsentry_sentry_go/transport.go:255 +0x5f goroutine 29 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc00106a750, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:190 +0x725 goroutine 30 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1e750, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 31 [select, 715 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1e8f0, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 32 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1e9c0, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 33 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1ea90, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 82 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1eb60, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 83 [select, 225 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1edd0, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 84 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1eea0, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 85 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1ef70, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 86 [select, 685 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1f040, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 87 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1f110, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 88 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc000e1f1e0, {0x63c8510, 0xc000a48d00}) github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x85 created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig github.com/cockroachdb/cockroach/pkg/util/log/flags.go:293 +0x1065 goroutine 37 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778690}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0xc00020f8f0, 0x40775a0}}, 0xc000510fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x51c9a38) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 38 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc0017788d0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc000690fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 39 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778870}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0xc000a7ec60}}, 0xc000694fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000a48d00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 40 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778a50}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc0009a8fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 41 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc000d0c480}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x403030203020203, 0x0}}, 0xc0009a97a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 42 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778750}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc0009a9fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 43 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc0017786f0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc0009aa7a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 44 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778810}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc0009aafa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 45 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc0017787b0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc0009ab7a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 46 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778990}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc0017d37a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x3) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 47 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc0017789f0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc0017d3fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 48 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778930}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc0017d47a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 49 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc00069a0c0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x7a, 0x7c}}, 0xc0017d4fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x7f) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 98 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778ab0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0xc000a7ec60}}, 0xc0006917a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000a48d00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 99 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc000d0c630}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00068f7a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 100 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc000d0c750}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00068ffa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 101 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:302 +0xe9 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:297 +0x85 goroutine 102 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 103 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 104 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 109 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc00ce9b4d0}, 0xc00175e910, {0xc011896258, 0x12}, 0xc00876b080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc00ce9b4d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 309 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc0199ccdc0, {0x63c85b8, 0xc00f726960}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:475 +0x346 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809201 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020eef040, {0x63c85b8, 0xc02f9d3d40}, 0xc02a5754b8, 0x1, 0xc0247290a8, 0x519bbc, 0xc0465c0840, 0xc04c4ebb60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02f9d3d40}, 0x120, 0xc024729230, 0x465c07e0, 0x0, 0x660069, 0xc0465c0840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020eef040, {0x63c85b8, 0xc02f9d3d40}, 0xc0465c07e0, {{{{0x0, 0x0}, {0xc058bc9680, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02f9d3d40}, 0x6352410, 0x450cb130) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc020df94f0, {0x63c85b8, 0xc02f9d3d40}, {0xc0465c06c0, {0x1743011063a1674b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc020df94f0, {0x63c85b8, 0xc02f9d3d40}, 0xc0cd42bc20, 0xc0247299f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02f9d3d40}, 0x660089, {0xc0465c06c0, {0x1743011063a1674b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4a000, {0x63c85b8, 0xc02f9d3d40}, 0xc00c026400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063a1674b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063a1674b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f9d3a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f9d3a40}, {0xc02472b3f8, 0x660089}, 0xc02472b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f9d3a40}, {0x3}, 0xc00c026200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f9d3980}, 0xc00c026200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}, 0xc079f17300, 0xc086c7a048) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}, 0x3, 0xc007ec6b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02472b980}}}, {0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}, 0xc079f17300, 0xc007ec6b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f9d3980}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f9d3980}, {0x40, 0xc02472bb70}, 0xc02472bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f9d3980}, {0x4e9d7a0, 0xc00c026200}, 0xc079f17300, 0xc086c7a048) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f9d3980}, 0xc06754ac00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0465c05a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0465c05a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809303 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f7a4dc0, {0x63c85b8, 0xc039fcd860}, 0xc03783da88, 0x1, 0xc0269670a8, 0x519bbc, 0xc06cf61020, 0xc077ef76e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc039fcd860}, 0x120, 0xc026967230, 0x6cf60fc0, 0x0, 0x660069, 0xc06cf61020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f7a4dc0, {0x63c85b8, 0xc039fcd860}, 0xc06cf60fc0, {{{{0x0, 0x0}, {0xc046250000, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc039fcd860}, 0x6352410, 0x43391448) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f663040, {0x63c85b8, 0xc039fcd860}, {0xc06cf60ea0, {0x174301105af607c5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f663040, {0x63c85b8, 0xc039fcd860}, 0xc0489f4320, 0xc0269679f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc039fcd860}, 0x660089, {0xc06cf60ea0, {0x174301105af607c5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f7c3000, {0x63c85b8, 0xc039fcd860}, 0xc09d60b200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105af607c5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105af607c5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039fcd7a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039fcd7a0}, {0xc0269693f8, 0x660089}, 0xc026969400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039fcd7a0}, {0x342d346231352d32}, 0xc09d60b000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039fcd710}, 0xc09d60b000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}, 0xc028ad4000, 0xc045220810) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}, 0x3, 0xc040ec5d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026969980}}}, {0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}, 0xc028ad4000, 0xc040ec5d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc039fcd710}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039fcd710}, {0x40, 0xc026969b08}, 0xc026969ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc039fcd710}, {0x4e9d7a0, 0xc09d60b000}, 0xc028ad4000, 0xc045220810) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc039fcd710}, 0xc043cdd5c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06cf60a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06cf60a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 308 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc0199ccc60, {0x63c85b8, 0xc00f726900}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:475 +0x346 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 122 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 307 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc0199ccb00, {0x63c85b8, 0xc00f7268a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:475 +0x346 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28388 [runnable]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00baca9b0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc014b027e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 219809839 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc011ca7400, {0x63c85b8, 0xc031888c60}, 0xc0387df600, 0xc049577180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc011ca7400, {0x63c85b8, 0xc031888c60}, 0xc0387df600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067f5000f, 0x0, 0x0}, 0xc0152d4470, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067f5000f, 0x0, 0x0}, 0xc0152d4470, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031888b70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031888b70}, {0xc01ca4b3f8, 0x660089}, 0xc01ca4b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031888b70}, {0x0}, 0xc0387df300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031888ab0}, 0xc0387df300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}, 0xc012c81e20, 0xc06c98d4b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}, 0x3, 0xc062a51f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01ca4b980}}}, {0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}, 0xc012c81e20, 0xc062a51f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc031888ab0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031888ab0}, {0x40, 0xc01ca4bb70}, 0xc01ca4bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc031888ab0}, {0x4e9d7a0, 0xc0387df300}, 0xc012c81e20, 0xc06c98d4b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc031888ab0}, 0xc01730ed80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06118cd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06118cd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 329 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa8f20, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc011a32280, 0xc0034b8000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc011a32280, {0xc0034b8000, 0x17d0, 0x17d0}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc011a32280, {0xc0034b8000, 0xc0034b8005, 0x62}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000b2f8c8, {0xc0034b8000, 0x6, 0xc012bd0788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc039707b78, {0xc0034b8000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0105db078, {0x631b900, 0xc039707b78}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0105dae00, {0x63290e0, 0xc000b2f8c8}, 0x7fe424b20d40) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0105dae00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0105dae00, {0xc0143e4000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc013500900, {0xc019575000, 0x9, 0xc043402db0}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc013500900}, {0xc019575000, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc019575000, 0x9, 0xc043402cf0}, {0x631b600, 0xc013500900}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc019574fc0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00d1125a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 398 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc011eb2740) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 219810386 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc023d7c800, {0x63c85b8, 0xc076c82cf0}, 0xc08ad6e300, 0xc0c36a9680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023d7c800, {0x63c85b8, 0xc076c82cf0}, 0xc08ad6e300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cbaa387, 0x0, 0x0}, 0xc03905a610, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cbaa387, 0x0, 0x0}, 0xc03905a610, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076c82c60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076c82c60}, {0xc026f033f8, 0x660089}, 0xc026f03400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076c82c60}, {0x7fe3b7b23a40}, 0xc08ad6e100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076c82bd0}, 0xc08ad6e100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}, 0xc07252a740, 0xc03d2720a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}, 0x3, 0xc073fea340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026f03980}}}, {0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}, 0xc07252a740, 0xc073fea340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076c82bd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076c82bd0}, {0x40, 0xc026f03b70}, 0xc026f03ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076c82bd0}, {0x4e9d7a0, 0xc08ad6e100}, 0xc07252a740, 0xc03d2720a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076c82bd0}, 0xc04d652a20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0588e1e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0588e1e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 97 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1({0x63c85b8, 0xc00df9f680}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1385 +0x26f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 130 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func2({0xc0017aa900, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:541 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 60 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2({0xc01956c300, 0x63c8501}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 306 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1({0x63c85b8, 0xc00df9f6b0}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1420 +0x2ff github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 131 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1() github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:241 +0x125 created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:238 +0xb4 goroutine 132 [select]: github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1({0xc0017aac00, 0x0}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/infostore.go:192 +0x8a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 133 [select]: github.com/cockroachdb/cockroach/pkg/ccl/utilccl.UpdateMetricOnLicenseChange.func2({0x63c85b8, 0xc007b7a360}) github.com/cockroachdb/cockroach/pkg/ccl/utilccl/license_check.go:159 +0x145 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 134 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x8c created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x425 goroutine 135 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:425 +0x1dc created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:398 +0x5b goroutine 136 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x8c created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x425 goroutine 137 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:425 +0x1dc created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:398 +0x5b goroutine 138 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x8c created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x425 goroutine 139 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:425 +0x1dc created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:398 +0x5b goroutine 140 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x8c created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x425 goroutine 141 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:425 +0x1dc created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:398 +0x5b goroutine 142 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x8c created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x425 goroutine 143 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:425 +0x1dc created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:398 +0x5b goroutine 144 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:302 +0xe9 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:297 +0x85 goroutine 145 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778870}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a9057a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 146 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778930}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a905fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 147 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc0017787b0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x6342548, 0xc00a86d3e0}}, 0xc00a9067a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00a86d3e0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 148 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778b70}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x6342548, 0xc00a86d500}}, 0xc00a906fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00a86d500) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 149 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc007d685d0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x6342548, 0xc00a86d680}}, 0xc00a9077a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00a86d680) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 150 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778ab0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x6342548, 0xc00a86d740}}, 0xc00a907fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00a86d740) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 151 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778b10}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a9007a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 152 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc0017789f0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a900fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 153 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778c30}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a9017a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 154 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778c90}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a901fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 155 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778cf0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a9027a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 156 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778bd0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a902fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 157 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc007d68630}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a9037a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 158 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778d50}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x0, 0x0}}, 0xc00a903fa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 159 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc001778db0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x6342548, 0xc010f978c0}}, 0xc010ffe7a8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc010f978c0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 160 [chan receive, 716 minutes]: github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x63c85b8, 0xc007d68690}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:300 +0x5b runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167180, 0x6342548, 0xc010f97920}}, 0xc010ffefa8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc010f97920) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:299 +0x58 created by github.com/cockroachdb/pebble.(*tableCacheShard).init github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:291 +0xd1 goroutine 161 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 162 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 163 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 164 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 28226 [select]: github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1({0x63c85b8, 0xc02ca0c5a0}) github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1151 +0x2d9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 322 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00d1121e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 166 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 167 [sync.Cond.Wait, 716 minutes]: sync.runtime_notifyListWait(0xc0001e14e0, 0x0) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x47737c0) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:193 github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc0001e1400, {0x47e80c0, 0x9bf6f48}) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:396 +0x479 runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167060, 0x0, 0x0}}, 0xc01954ffc0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble/record.NewLogWriter.func2() github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:314 +0x5c created by github.com/cockroachdb/pebble/record.NewLogWriter github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:313 +0x329 goroutine 2530 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc016ce81e0, 0xc0181fa000, {0x643dbd8, 0xc015db28a0}, 0x20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc018d62150}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 58 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1({0x63c8510, 0xc0008a8680}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:116 +0x15a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 170 [select]: github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startEventIntake.func3({0xc00c99db00, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:188 +0xb1 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 171 [select]: github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startResolver.func1({0x63c85b8, 0xc00cbb0480}) github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:219 +0x219 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 172 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe8300, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 173 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe8600, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 174 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe8900, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 175 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe8c00, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 176 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe8f00, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 177 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe9200, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 194 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe9500, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 195 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe9800, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 196 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cbe9b00, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 197 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cc08000, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 198 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cc08300, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 199 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cc08600, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 200 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cc08900, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 201 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cc08c00, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 202 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cc08f00, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 203 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1({0xc00cc09200, 0x6342548}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:113 +0x10c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 204 [select]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x63c85b8, 0xc00cbb07b0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:139 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 205 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x63c85b8, 0xc00cbb07e0}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:139 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 206 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x63c85b8, 0xc00cbb0810}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:139 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 207 [select, 3 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x63c85b8, 0xc00cbb0840}) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:139 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 208 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc01f834b18) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01978c0f0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01978c0f0, {0x63c8510, 0xc019792000}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x63c85b8, 0xc01978c030}, {0xc01f834c98, 0x2, 0x4000000000000000}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:193 +0x91 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00c9418c0, {0x63c85b8, 0xc01978c030}, 0xc00cbb08a0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:291 +0x5de github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x63c85b8, 0xc00cbb09c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:215 +0x339 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 210 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc003581cf8) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc019550180) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc019550180, {0x63c8510, 0xc0008a8680}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00167cf00, {0x63c8510, 0xc019792000}, {0xc00cbb0930, 0x1, 0xc00cbb08a0}, {0xc00c9418c0, 0x19788000, 0xc0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:139 +0x625 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc00cbb09c0, {0x63c8510, 0xc019792000}, {0xc00cbb0930, 0x63c85b8, 0xc00cbb09c0}, {0x707000000000000, 0x63c85b8, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:78 +0x45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x63c8510, 0xc019792000}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:285 +0x7a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 211 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00c9418c0, {0x63c8510, 0xc019792000}, 0xc00cbb08a0, 0xc019790060) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:331 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x63c8510, 0xc019792000}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:288 +0x38 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 1289 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab974f0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc021e72280, 0xc01df9a480, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc021e72280, {0xc01df9a480, 0x205, 0x205}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc021e72280, {0xc01df9a480, 0x0, 0x0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0211f0d90, {0xc01df9a480, 0xc025bd8bf0, 0x78}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc021e6f740, {0xc01df9a480, 0x57c655, 0x205}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc012f03140, {0xc01df9a480, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00e057778, {0x631b900, 0xc012f03140}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00e057500, {0x7fe42ab59310, 0xc021e6f740}, 0xc0140e5318) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00e057500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00e057500, {0xc022b90000, 0x1000, 0x52bee4}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f net/http.(*connReader).Read(0xc022b8e090, {0xc022b90000, 0x1000, 0x1000}) GOROOT/src/net/http/server.go:780 +0x16d bufio.(*Reader).fill(0xc021e7f2c0) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadSlice(0xc021e7f2c0, 0x0) GOROOT/src/bufio/bufio.go:360 +0x2f bufio.(*Reader).ReadLine(0xc021e7f2c0) GOROOT/src/bufio/bufio.go:389 +0x27 net/textproto.(*Reader).readLineSlice(0xc05be83b00) GOROOT/src/net/textproto/reader.go:57 +0x99 net/textproto.(*Reader).ReadLine(...) GOROOT/src/net/textproto/reader.go:38 net/http.readRequest(0xc0140e5a20) GOROOT/src/net/http/request.go:1029 +0x79 net/http.(*conn).readRequest(0xc021e763c0, {0x63c8510, 0xc021e3be40}) GOROOT/src/net/http/server.go:966 +0x225 net/http.(*conn).serve(0xc021e763c0, {0x63c85b8, 0xc01fade570}) GOROOT/src/net/http/server.go:1856 +0x865 created by net/http.(*Server).Serve GOROOT/src/net/http/server.go:3034 +0x4e8 goroutine 270 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00d173240) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 209 [select, 59 minutes]: github.com/cockroachdb/cockroach/pkg/server.startPurgeOldSessions.func1({0xc00cc20600, 0x0}) github.com/cockroachdb/cockroach/pkg/server/purge_auth_session.go:69 +0x1e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 242 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1() github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:263 +0x165 created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps github.com/cockroachdb/cockroach/pkg/util/hlc/hlc.go:255 +0x145 goroutine 243 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2({0xc00cc20c00, 0x0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:113 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 244 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*httpServer).start.func1({0x63c85b8, 0xc00cbb1a10}) github.com/cockroachdb/cockroach/pkg/server/server_http.go:226 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 245 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa9970, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00cb72700, 0xc000097000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc00cb72700) GOROOT/src/internal/poll/fd_unix.go:402 +0x22c net.(*netFD).accept(0xc00cb72700) GOROOT/src/net/fd_unix.go:173 +0x35 net.(*TCPListener).accept(0xc00c985e18) GOROOT/src/net/tcpsock_posix.go:140 +0x28 net.(*TCPListener).Accept(0xc00c985e18) GOROOT/src/net/tcpsock.go:262 +0x3d github.com/cockroachdb/cmux.(*cMux).Serve(0xc00cc2a480) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 +0xa9 github.com/cockroachdb/cockroach/pkg/server.(*httpServer).start.func2({0xc00cc21200, 0x0}) github.com/cockroachdb/cockroach/pkg/server/server_http.go:243 +0x22 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 246 [chan receive, 716 minutes]: github.com/cockroachdb/cmux.muxListener.Accept(...) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 net/http.(*Server).Serve(0xc019574000, {0x6367a10, 0xc00c985e48}) GOROOT/src/net/http/server.go:3002 +0x394 github.com/cockroachdb/cockroach/pkg/server.(*httpServer).start.func3({0xc00cc21500, 0x0}) github.com/cockroachdb/cockroach/pkg/server/server_http.go:264 +0x13c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 247 [chan receive]: github.com/cockroachdb/cmux.muxListener.Accept(...) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 crypto/tls.(*listener).Accept(0xc00c985e90) GOROOT/src/crypto/tls/tls.go:66 +0x2d net/http.(*Server).Serve(0xc00cb6c700, {0x634ac10, 0xc00c985e90}) GOROOT/src/net/http/server.go:3002 +0x394 github.com/cockroachdb/cockroach/pkg/server.(*httpServer).start.func4({0xc00cc21800, 0x0}) github.com/cockroachdb/cockroach/pkg/server/server_http.go:277 +0x28 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 248 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func3({0xc00cc21b00, 0xc000167060}) github.com/cockroachdb/cockroach/pkg/server/start_listen.go:108 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 249 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func1({0xc00cf4a000, 0x0}) github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:86 +0x37 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 250 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*loopbackListener).Accept(0xc00ce509c0) github.com/cockroachdb/cockroach/pkg/server/loopback.go:49 +0xa5 google.golang.org/grpc.(*Server).Serve(0xc000f02e00, {0x6353f40, 0xc00ce509c0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:780 +0x477 github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func2({0xc00cf4a300, 0x0}) github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:94 +0x2b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 251 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00cc2ad80) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 214 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc01979e7d0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0197ec0c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 253 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func5({0x63c85b8, 0xc00ce50fc0}) github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:127 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 291 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc01979b6c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 62 [select]: net.(*pipe).read(0xc019546180, {0xc00b90b000, 0x4b3a, 0xc00b90b005}) GOROOT/src/net/pipe.go:159 +0x15b net.(*pipe).Read(0xc002d9bd06, {0xc00b90b000, 0x48578c, 0x1000000051cdcd8}) GOROOT/src/net/pipe.go:142 +0x25 crypto/tls.(*atLeastReader).Read(0xc0379d1b90, {0xc00b90b000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0197a2278, {0x631b900, 0xc0379d1b90}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0197a2000, {0x7fe456a34350, 0xc019546180}, 0x4b3a) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0197a2000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0197a2000, {0xc0110f2000, 0x8000, 0x459769}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc01957c240, {0xc019574120, 0x9, 0xc002d9bd18}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc01957c240}, {0xc019574120, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc019574120, 0x9, 0xc002d9bd6f}, {0x631b600, 0xc01957c240}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0195740e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00d112000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 254 [chan receive]: github.com/cockroachdb/cmux.muxListener.Accept(...) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 google.golang.org/grpc.(*Server).Serve(0xc000f02e00, {0x6367a10, 0xc00ce5a1f8}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:780 +0x477 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func5.1({0xc00cf4a900, 0x0}) github.com/cockroachdb/cockroach/pkg/server/start_listen.go:134 +0x28 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 255 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa9a60, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00cb73680, 0xc000085800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc00cb73680) GOROOT/src/internal/poll/fd_unix.go:402 +0x22c net.(*netFD).accept(0xc00cb73680) GOROOT/src/net/fd_unix.go:173 +0x35 net.(*TCPListener).accept(0xc00ce5a1c8) GOROOT/src/net/tcpsock_posix.go:140 +0x28 net.(*TCPListener).Accept(0xc00ce5a1c8) GOROOT/src/net/tcpsock.go:262 +0x3d github.com/cockroachdb/cmux.(*cMux).Serve(0xc00cc2ab80) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 +0xa9 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func5.2.1() github.com/cockroachdb/cockroach/pkg/server/start_listen.go:139 +0x22 sync.(*Once).doSlow(0xc0110d16e0, 0xc0110d1718) GOROOT/src/sync/once.go:68 +0xd2 sync.(*Once).Do(...) GOROOT/src/sync/once.go:59 github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func5.2({0xc00cf4ac00, 0x0}) github.com/cockroachdb/cockroach/pkg/server/start_listen.go:138 +0x55 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 106 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc00ceca680, {0x63c85b8, 0xc00ce9b050}, 0xc00102e4e0, {0x643dad0, 0xc0198c0120}, 0xc000d3ab40, 0xc008726ca0) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:371 +0x475 github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1({0x63c85b8, 0xc00ce9b050}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:125 +0x416 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 213 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000d0ee60, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0197ec060) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 215 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc01979a4e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 216 [select]: net.(*pipe).read(0xc019546200, {0xc0197e0000, 0x811, 0xc0197e0005}) GOROOT/src/net/pipe.go:159 +0x15b net.(*pipe).Read(0x6, {0xc0197e0000, 0x4bf403, 0x10000c03c92b508}) GOROOT/src/net/pipe.go:142 +0x25 crypto/tls.(*atLeastReader).Read(0xc0379d1ba8, {0xc0197e0000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc01957a278, {0x631b900, 0xc0379d1ba8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc01957a000, {0x7fe456a34350, 0xc019546200}, 0x811) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc01957a000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc01957a000, {0xc0197ee000, 0x8000, 0x81f010601}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc0197ec000, {0xc0197da2e0, 0x9, 0x18}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc0197ec000}, {0xc0197da2e0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0197da2e0, 0x9, 0xc0286640a8}, {0x631b600, 0xc0197ec000}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0197da2a0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc01979a4e0, 0x0, 0x0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc01979a4e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 290 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc01979f860, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00dce3200) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 107 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00870cf40) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 219807726 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fe85000, {0x63c85b8, 0xc07a4154d0}, 0xc0283edb00, 0xc0861d0fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fe85000, {0x63c85b8, 0xc07a4154d0}, 0xc0283edb00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011059e0a515, 0x0, 0x0}, 0xc08e62b2b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011059e0a515, 0x0, 0x0}, 0xc08e62b2b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a415440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a415440}, {0xc00e8213f8, 0x660089}, 0xc00e821400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a415440}, {0x7fe425757b58}, 0xc04a321f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a4153b0}, 0xc04a321f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}, 0xc06be601c0, 0xc030d2d3f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}, 0x3, 0xc064bac740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e821980}}}, {0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}, 0xc06be601c0, 0xc064bac740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a4153b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a4153b0}, {0x40, 0xc00e821b70}, 0xc00e821ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a4153b0}, {0x4e9d7a0, 0xc04a321f00}, 0xc06be601c0, 0xc030d2d3f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a4153b0}, 0xc0651ddb60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc071835440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc071835440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc01979eaf0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00cd75e00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 220 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc01979ab60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 221 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa9790, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0197c4500, 0xc0308ee000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0197c4500, {0xc0308ee000, 0xe213, 0xe213}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0197c4500, {0xc0308ee000, 0xd7b5, 0xc0197bcdc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01978a0b0, {0xc0308ee000, 0x7eda79, 0xc0308eea63}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0197ec480, {0xc0308ee000, 0x48578c, 0xe213}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0342fd830, {0xc0308ee000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0197a25f8, {0x631b900, 0xc0342fd830}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0197a2380, {0x7fe42ab59310, 0xc0197ec480}, 0xd7b5) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0197a2380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0197a2380, {0xc011256000, 0x8000, 0x10000665b000001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc00d1ec300, {0xc0197da4a0, 0x9, 0xc060d4e200}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc00d1ec300}, {0xc0197da4a0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0197da4a0, 0x9, 0x0}, {0x631b600, 0xc00d1ec300}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0197da460) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc01979ab60, 0x0, 0x0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc01979ab60}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 96 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3({0xc00d522000, 0xc01faead01}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/server.go:380 +0x49 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219798864 [runnable]: sync.runtime_SemacquireMutex(0xc093750690, 0x8f, 0x9bb12c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc093750538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc093750560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc040848d80, {0xffffffffffffff15, 0xc0937506c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc040848d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc040848d80, {0xc01eb58310, 0x32, 0x6e}, {0xc01d642b00, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b1ed5998, {0xc01eb58310, 0x32, 0x6e}, {0xc01d642b00, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b1ed42c0, {0xc01d642b00, 0x77, 0x6e}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b1ed42c0, {0xc01eb58310, 0x77, 0xc01d642b00}, {0xc01d642b00, 0xc08208e7e0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b1ed4000, {0xc01d642b00, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc067699328, {{0xc08208e7e0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0542ac500, {{0xc00f90e2a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc004ecfc00, {0x63c85b8, 0xc038bba930}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc038bba930}, {0x64a6560, 0xc0542ac500}, {0xc00f90e2a0, 0x7171717171717171, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc038bba930}, {0x6466ed8, 0xc067699080}, {0xc00f90e2a0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07216d080}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc027cfed00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020799800, {0x63c85b8, 0xc038bba930}, 0xc027cfed00, 0xc07076b2c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020799800, {0x63c85b8, 0xc038bba930}, 0xc027cfed00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc038bba8a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc038bba8a0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc038bba8a0}, {0xc00102e4e0}, 0xc027cfec00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc038bba840}, 0xc027cfec00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc038bba750}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 258 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000d0f7c0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00d89c600) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 259 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc019572680) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 260 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa96a0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc019546400, 0xc021f9a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc019546400, {0xc021f9a000, 0xcc52, 0xcc52}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc019546400, {0xc021f9a000, 0xc1a5, 0xc00d320140}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000f063c8, {0xc021f9a000, 0x7eda79, 0xc021f9aab2}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01957d0e0, {0xc021f9a000, 0x4bf403, 0xcc52}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a90450, {0xc021f9a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0197a2cf8, {0x631b900, 0xc043a90450}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0197a2a80, {0x7fe42ab59310, 0xc01957d0e0}, 0xc1a5) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0197a2a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0197a2a80, {0xc01131c000, 0x8000, 0x10000da83000001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc01957d5c0, {0xc0195742e0, 0x9, 0xc01014ac00}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc01957d5c0}, {0xc0195742e0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0195742e0, 0x9, 0x0}, {0x631b600, 0xc01957d5c0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0195742a0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc019572680, 0x10bed2f, 0xc00cf4a300) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc019572680}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 111 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc011976000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 219809192 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc02b75e660}, 0xc09b41a500, 0xc07298bb80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02b75e660}, 0xc09b41a500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106346bdcd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106346bdcd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b75e450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b75e450}, {0xc00f5693f8, 0x660089}, 0xc00f569400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b75e450}, {0x0}, 0xc09b41a200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f3f7ec0}, 0xc09b41a200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}, 0xc0707cfe00, 0xc056a74120) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}, 0x3, 0xc053216d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f569980}}}, {0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}, 0xc0707cfe00, 0xc053216d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f3f7ec0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f3f7ec0}, {0x40, 0xc00f569b70}, 0xc00f569ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f3f7ec0}, {0x4e9d7a0, 0xc09b41a200}, 0xc0707cfe00, 0xc056a74120) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f3f7ec0}, 0xc07a834fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01e75fd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01e75fd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 292 [runnable]: internal/poll.runtime_pollWait(0x7fe456aa94c0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0197c4b00, 0xc003890000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0197c4b00, {0xc003890000, 0xf249, 0xf249}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0197c4b00, {0xc003890000, 0xf249, 0xc0197bd7c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01978a938, {0xc003890000, 0x7eda79, 0xc003890005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00ddf2600, {0xc003890000, 0x7fe422ad19b0, 0xf249}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02b06ab10, {0xc003890000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00151f3f8, {0x631b900, 0xc02b06ab10}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00151f180, {0x7fe42ab59310, 0xc00ddf2600}, 0xf249) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00151f180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00151f180, {0xc01141c000, 0x8000, 0x5011000001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc00ddf2ae0, {0xc0197da660, 0x9, 0x5023000001}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc00ddf2ae0}, {0xc0197da660, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0197da660, 0x9, 0x50}, {0x631b600, 0xc00ddf2ae0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0197da620) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc01979b6c0, 0xa6f125, 0xa6ca8f) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 272 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc00e0364e0}, 0xc00e024c30, {0xc0002f2078, 0x11}, 0xc00e0169c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc00e0364e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 216536974 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc061219b50, 0xe2) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc020213500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc020213b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04f01d300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04f01d300, {0x63c8510, 0xc061219b80}, 0xc051ec5dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03020f500) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc061219b80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03020f500) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 406 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa9010, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00dff8c00, 0xc0012f5500, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00dff8c00, {0xc0012f5500, 0x924, 0x924}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00dff8c00, {0xc0012f5500, 0xc0012f5505, 0xd9}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000b2f878, {0xc0012f5500, 0x6, 0xc01117b788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc0ce34e0c0, {0xc0012f5500, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0105dacf8, {0x631b900, 0xc0ce34e0c0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0105daa80, {0x63290e0, 0xc000b2f878}, 0xc) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0105daa80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0105daa80, {0xc014128000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc013410b40, {0xc00f400740, 0x9, 0xc0676f7ec0}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc013410b40}, {0xc00f400740, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00f400740, 0x9, 0xc0676f7e00}, {0x631b600, 0xc013410b40}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00f400700) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0011ba3c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 408 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc012856a00) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 323 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa93d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc000884d80, 0xc01f4e6000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000884d80, {0xc01f4e6000, 0xa768, 0xa768}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000884d80, {0xc01f4e6000, 0xc01f4e6005, 0x62}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000f06568, {0xc01f4e6000, 0x6, 0xc000491788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc06faab7b8, {0xc01f4e6000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00e0565f8, {0x631b900, 0xc06faab7b8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00e056380, {0x63290e0, 0xc000f06568}, 0xc) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00e056380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00e056380, {0xc01162a000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc00e063680, {0xc019574660, 0x9, 0xc073a93c80}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc00e063680}, {0xc019574660, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc019574660, 0x9, 0xc073a93bc0}, {0x631b600, 0xc00e063680}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc019574620) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00d1121e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 324 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00e025220, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00e0636e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 219805398 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc078635470}, 0xc08b703a00, 0xc05ad63180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc078635470}, 0xc08b703a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041ec4a1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041ec4a1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078635290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078635290}, {0xc07c0cb3f8, 0x660089}, 0xc07c0cb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078635290}, {0x7fe3f3396710}, 0xc08b703800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078635200}, 0xc08b703800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}, 0xc0a0c54c20, 0xc0525c1b90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}, 0x3, 0xc06b90b100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07c0cb980}}}, {0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}, 0xc0a0c54c20, 0xc06b90b100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc078635200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078635200}, {0x40, 0xc07c0cbb70}, 0xc07c0cbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}, 0x20301a, 0x20301a) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc078635200}, {0x4e9d7a0, 0xc08b703800}, 0xc0a0c54c20, 0xc0525c1b90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc078635200}, 0xc023a085a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ad5a360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ad5a360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 330 [runnable]: syscall.Syscall(0x1, 0x520, 0xc029dc5800, 0x8d) GOROOT/src/syscall/asm_linux_amd64.s:20 +0x5 syscall.write(0xc011a32280, {0xc029dc5800, 0x0, 0x0}) GOROOT/src/syscall/zsyscall_linux_amd64.go:915 +0x4d syscall.Write(...) GOROOT/src/syscall/syscall_unix.go:214 internal/poll.ignoringEINTRIO(...) GOROOT/src/internal/poll/fd_unix.go:582 internal/poll.(*FD).Write(0xc011a32280, {0xc029dc5800, 0x8d, 0x4800}) GOROOT/src/internal/poll/fd_unix.go:275 +0x36e net.(*netFD).Write(0xc011a32280, {0xc029dc5800, 0x0, 0x49afb40}) GOROOT/src/net/fd_posix.go:74 +0x29 net.(*conn).Write(0xc000b2f8c8, {0xc029dc5800, 0x0, 0x0}) GOROOT/src/net/net.go:195 +0x45 crypto/tls.(*Conn).write(0xc0105dae00, {0xc029dc5800, 0x5, 0x4800}) GOROOT/src/crypto/tls/conn.go:913 +0x108 crypto/tls.(*Conn).writeRecordLocked(0xc0105dae00, 0x17, {0xc029220000, 0x77, 0x10000}) GOROOT/src/crypto/tls/conn.go:981 +0x351 crypto/tls.(*Conn).Write(0x0, {0xc029220000, 0x77, 0x10000}) GOROOT/src/crypto/tls/conn.go:1152 +0x405 google.golang.org/grpc/internal/transport.(*bufWriter).Flush(0xc0129caaa0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http_util.go:366 +0x6c google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc013500960) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:574 +0x6e google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 112 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa92e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc01f7e8900, 0xc00d822000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc01f7e8900, {0xc00d822000, 0x326c, 0x326c}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc01f7e8900, {0xc00d822000, 0xc00d8223eb, 0x398}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000b2f5e0, {0xc00d822000, 0x6, 0xc011328788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc021b9b698, {0xc00d822000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0105da5f8, {0x631b900, 0xc021b9b698}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0105da380, {0x63290e0, 0xc000b2f5e0}, 0x2e86) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0105da380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0105da380, {0xc012c0e000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc01191d800, {0xc0105b4580, 0x9, 0xc011328cb8}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc01191d800}, {0xc0105b4580, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0105b4580, 0x9, 0xc00ce9bd40}, {0x631b600, 0xc01191d800}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0105b4540) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc011976000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 113 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00175ef00, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc01191d860) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 338 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 339 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 340 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0198ac280, {0xc0198aa238, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:191 +0xaa google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0198ac280, {0xc0198aa238, 0xc0198ae030, 0xc01f832950}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:171 +0x147 google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0198b0510, {0xc0198aa238, 0xc01f8329c8, 0xa764c7}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc0198b0510}, {0xc0198aa238, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0198f2240, {0xc0198aa238, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc0198aa228, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x47a9480, 0xc0198f2240, {0x0, 0x0}, 0x7fffffff, 0xc01f832c80, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0xc01f832bf0, {0x7fe42ab594c0, 0xc000808f40}, 0x1704030100, {0x0, 0x0}, {0x4cf77a0, 0xc0689aefa0}, 0x479aa80, 0xc01f832c80, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*csAttempt).recvMsg(0xc0198f4000, {0x4cf77a0, 0xc0689aefa0}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:983 +0x2b0 google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0198f2000, 0xc01f832db8, 0xc01f832d88) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0198f2000, {0x4cf77a0, 0xc0689aefa0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc0198da0e0, {0x4cf77a0, 0xc0689aefa0}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc0198c0120) github.com/cockroachdb/cockroach/pkg/gossip/bazel-out/k8-opt/bin/pkg/gossip/gossip_go_proto_/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:636 +0x4c github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0xc007b77ec0, {0x643dad0, 0xc0198c0120}, 0xc00ceca680, {0x63c85b8, 0xc0198b0660}, 0x0) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:324 +0x69 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2({0x63c85b8, 0xc0198b0660}) github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:339 +0x91 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803123 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff35d00, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc050408f80, 0xc043869000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc050408f80, {0xc043869000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc050408f80, {0xc043869000, 0x735, 0xc04f7ae500}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03bd0f680, {0xc043869000, 0x7eda79, 0xc043869005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0aaac2300, {0xc043869000, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc030f86d98, {0xc043869000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc075a1c978, {0x631b900, 0xc030f86d98}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc075a1c700, {0x7fe42ab59310, 0xc0aaac2300}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc075a1c700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc075a1c700, {0xc006895000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc028a61b90, {0xc006895000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0437a3588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0437a3588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0437a3b60, {0x634aac0, 0xc0437a3588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0437a3500, 0xc040f8e7b0, 0xc00b0ed42b, 0xc00b0ed42d, {0x635bec0, 0xc045e874d0}, 0xc000d6ae00, 0xc0631e7588, 0xc00b0ed429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0437a3500, {0x63c85b8, 0xc039067740}, 0xc040f8e7a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0ddb04009, 0x8}}, 0x0, 0xc0390676b0, 0xc039067710, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc039067740}, {0x643d810, 0xc0aaac2300}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc027174ba0}, {0x643d810, 0xc0aaac2300}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc027174ba0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 400 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc011ec2db0}, 0xc011eaccd0, {0xc011896420, 0x12}, 0xc011ea8780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc011ec2db0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 328 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00d1125a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 405 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0011ba3c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 2528 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0119761e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 219801073 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc089972548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0899725b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc09475b1f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0a20c8e00, {0xc01c512300, 0x2b, 0x74}, {0xc03eec9b20, 0x2b, 0x6e}, 0x1, 0xb1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc09475b088, {0xc01c512300, 0x1, 0x6e}, {0xc03eec9b20, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09475b088, {0xc01c512300, 0x2b, 0x74}, {0xc03eec9b20, 0x2b, 0x6e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09475a7c0, {0xc03eec9b20, 0x6e, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09475a7c0, {0xc01c512300, 0x2b, 0xc03eec9b20}, {0xc03eec9b20, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09475a500, {0xc03eec9b20, 0x2b, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06e37db88, {{0xc051eb7bf0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc058b30d00, {{0xc051eb7bf0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc03e8a3c00, {0x63c85b8, 0xc060591c80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc060591c80}, {0x64a6560, 0xc058b30d00}, {0xc051eb7bf0, 0xedb7a3892, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc060591c80}, {0x6466ed8, 0xc06e37db80}, {0xc051eb7bf0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0266624f8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc00719fd00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020d8d000, {0x63c85b8, 0xc060591c80}, 0xc00719fd00, 0xc0339ada40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020d8d000, {0x63c85b8, 0xc060591c80}, 0xc00719fd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060591bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060591bf0}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060591bf0}, {0xc00102e4e0}, 0xc00719fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc060591b90}, 0xc00719fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618caab0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806470 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa02d8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05f041400, 0xc05b379800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05f041400, {0xc05b379800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05f041400, {0xc05b379800, 0x735, 0xc0936c0dc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03a07efc0, {0xc05b379800, 0x7eda79, 0xc05b379805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03bc953e0, {0xc05b379800, 0xaeb229f18093de9, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc044e34c48, {0xc05b379800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc08d575078, {0x631b900, 0xc044e34c48}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc08d574e00, {0x7fe42ab59310, 0xc03bc953e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc08d574e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc08d574e00, {0xc017ef1000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc08ecfeeb8, {0xc017ef1000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc01a6dcb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc01a6dcb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc01a6dd0e0, {0x634aac0, 0xc01a6dcb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc01a6dca80, 0xc051ffda50, 0xc074ccb42b, 0xc074ccb42d, {0x635bec0, 0xc08456e2d0}, 0xc000d6ae00, 0xc07b9f9e28, 0xc074ccb429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc01a6dca80, {0x63c85b8, 0xc084c611a0}, 0xc051ffda40, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc01b1a8037, 0x8}}, 0x0, 0xc084c61140, 0xc084c61170, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc084c611a0}, {0x643d810, 0xc03bc953e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07a8c3260}, {0x643d810, 0xc03bc953e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07a8c3260}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812198 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a65d400, {0x63c85b8, 0xc03d0ea9c0}, 0xc047c09080, 0x1, 0xc0330190a8, 0x519bbc, 0xc049b48600, 0xc0aa155140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03d0ea9c0}, 0x1e0, 0xc033019230, 0x49b485a0, 0x0, 0x660069, 0xc049b48600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a65d400, {0x63c85b8, 0xc03d0ea9c0}, 0xc049b485a0, {{{{0x0, 0x0}, {0xc067748480, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03d0ea9c0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a5d7b30, {0x63c85b8, 0xc03d0ea9c0}, {0xc020c17560, {0x174301107ea5be60, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a5d7b30, {0x63c85b8, 0xc03d0ea9c0}, 0xc03e98b720, 0xc0330199f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03d0ea9c0}, 0x660089, {0xc020c17560, {0x174301107ea5be60, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a26f400, {0x63c85b8, 0xc03d0ea9c0}, 0xc059187c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107ea5be60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107ea5be60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03d0ea930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d0ea930}, {0xc03301b3f8, 0x660089}, 0xc03301b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03d0ea930}, {0xc0015221e0}, 0xc059187a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03d0ea840}, 0xc059187a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}, 0xc078fbc700, 0xc024450180) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}, 0x3, 0xc039953d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03301b980}}}, {0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}, 0xc078fbc700, 0xc039953d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03d0ea840}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d0ea840}, {0x40, 0xc03301bb70}, 0xc03301bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03d0ea840}, {0x4e9d7a0, 0xc059187a00}, 0xc078fbc700, 0xc024450180) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03d0ea840}, 0xc0575590e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0688be6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0688be6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 407 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc011ff9400, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc013410c00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 1035 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0215a9590, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc021405e00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 410 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc013466300}, 0xc011ff96d0, {0xc011896e88, 0x12}, 0xc011fe57a0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc013466300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 411 [select]: github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal({{0x63c85b8, 0xc01de8ab10}, 0xc018d75200, 0xc018d75260}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:694 +0xbf github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:723 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc00167cf00, {0x63c8510, 0xc0008a8680}, {{0xc00bc1ea60, 0x1, 0x8}, {0xc00bc1ea68, 0x1, 0x8}}, {0x1742d9fc019bb7a1, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:519 +0xd84 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc00167cf00, {0x63c8510, 0xc0008a8680}, 0xc0011c7440, {{0xc00bc1ea60, 0x1, 0x8}, {0xc00bc1ea68, 0x1, 0x8}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:336 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1({0x63c8510, 0xc0008a8680}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:120 +0xbe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 787 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0195f97c0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0180a1440) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 219804952 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc05309a390}, 0xc031050db0, 0x1, 0x708017d9f9c71b, 0x37e11d600, 0xc053d73aa0, 0xc0552e98c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc05309a390}, 0x1e0, 0xc078567230, 0x53d73a40, 0xc01faebd40, 0x18, 0xc053d73aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc05309a390}, 0xc053d73a40, {{{{0x0, 0x0}, {0xc05710c240, 0x4f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc05309a390}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc05309a390}, {0xc070306ea0, {0x174301103c0ea14d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc05309a390}, 0xc07cd219a0, 0xc0785679f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05309a390}, 0x660089, {0xc070306ea0, {0x174301103c0ea14d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc05309a390}, 0xc06457ae00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103c0ea14d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103c0ea14d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05309a300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05309a300}, {0xc0785693f8, 0x660089}, 0xc078569400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05309a300}, {0xc0015221e0}, 0xc06457ab00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05309a270}, 0xc06457ab00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}, 0xc06649aa60, 0xc031050d80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}, 0x3, 0xc05969b3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078569980}}}, {0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}, 0xc06649aa60, 0xc05969b3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05309a270}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05309a270}, {0x40, 0xc078569b70}, 0xc078569ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}, 0x203016, 0x203016) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05309a270}, {0x4e9d7a0, 0xc06457ab00}, 0xc06649aa60, 0xc031050d80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05309a270}, 0xc031305080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc070306900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc070306900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806992 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc0606c3320}, {0xc0ae5d6f20, 0x2, 0xc00f31ce58}, {0x0, {0x1743010fedf978ad, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc0606c3320}, 0x1866f, 0xc08a2b0900, 0x18, 0xc038af4220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc0606c3320}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 333 [runnable]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0129cafa0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0133da480) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 334 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc019573860) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 335 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa8d40, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc019547880, 0xc003aa6000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc019547880, {0xc003aa6000, 0xf236, 0xf236}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc019547880, {0xc003aa6000, 0xf236, 0xc01352c500}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc012e3c808, {0xc003aa6000, 0x7eda79, 0xc003aa6005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc013500cc0, {0xc003aa6000, 0x7fe422ad19b0, 0xf236}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc060f8e720, {0xc003aa6000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00e057078, {0x631b900, 0xc060f8e720}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00e056e00, {0x7fe42ab59310, 0xc013500cc0}, 0xf236) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00e056e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00e056e00, {0xc014534000, 0x8000, 0x5353000001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc01355cb40, {0xc0195751c0, 0x9, 0x5343000001}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc01355cb40}, {0xc0195751c0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0195751c0, 0x9, 0x53}, {0x631b600, 0xc01355cb40}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc019575180) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc019573860, 0x10bed2f, 0xc00cc21500) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc019573860}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 219808229 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc022290510}, 0xc06e3d73e0, 0x1, 0xc00f4710a8, 0x519bbc, 0xc072f77b60, 0xc072f762a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc022290510}, 0x120, 0xc00f471230, 0x72f77b00, 0x0, 0x660069, 0xc072f77b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc022290510}, 0xc072f77b00, {{{{0x0, 0x0}, {0xc041e0d8c0, 0x39}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc022290510}, 0x6352410, 0x2eda49a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc022290510}, {0xc072f779e0, {0x174301105a7a2ce9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc022290510}, 0xc077a2c000, 0xc00f4719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022290510}, 0x660089, {0xc072f779e0, {0x174301105a7a2ce9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc022290510}, 0xc07babde00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a7a2ce9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a7a2ce9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022290390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022290390}, {0xc00f4733f8, 0x660089}, 0xc00f473400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022290390}, {0x7fe3f1e9ede8}, 0xc07babdc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022290300}, 0xc07babdc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}, 0xc078d8b820, 0xc014e58870) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}, 0x3, 0xc046a21500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f473980}}}, {0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}, 0xc078d8b820, 0xc046a21500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022290300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022290300}, {0x40, 0xc00f473b70}, 0xc00f473ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022290300}, {0x4e9d7a0, 0xc07babdc00}, 0xc078d8b820, 0xc014e58870) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022290300}, 0xc026c28600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc072f778c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc072f778c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218651900 [select, 2 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc06b277320) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0393974a0, {0x4cfcb60, 0xc06b276fc0}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6f) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc06b2770e0, 0xc058bcf698, 0xc058bcf668) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc06b2770e0, {0x4cfcb60, 0xc06b276fc0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc077fe2db0}, {0x4fbd2c0, 0x2}, {0x4cfca20, 0xc06b4b4800}, {0x4cfcb60, 0xc06b276fc0}, 0xc077fe2cc0, {0xc00c5825c0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc077fe2cc0}, {0x4fbd2c0, 0x7fe457d92f48}, {0x4cfca20, 0xc06b4b4800}, {0x4cfcb60, 0xc06b276fc0}, 0xc077fe2cc0, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x7fe457d92f48, {0x63c85b8, 0xc077fe2cc0}, {0x4fbd2c0, 0x0}, {0x4cfca20, 0xc06b4b4800}, {0x4cfcb60, 0xc06b276fc0}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*perReplicaClient).CollectChecksum(0xc0018569a8, {0x63c85b8, 0xc077fe2cc0}, 0x38, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:261 +0xc9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).collectChecksumFromReplica(_, {_, _}, {_, _, _, _}, {0xc1, 0x8b, 0x31, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_consistency.go:344 +0x13c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RunConsistencyCheck.func2({0x63c85b8, 0xc077fe2cc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_consistency.go:398 +0x14d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2563 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue.func2() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:352 +0x8c created by github.com/cockroachdb/cockroach/pkg/util/admission.makeWorkQueue github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:349 +0x425 goroutine 1132 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc020d410e0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0214040c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 788 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc01347c1a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 789 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab97e50, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc016ac1280, 0xc00640a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc016ac1280, {0xc00640a000, 0xd66e, 0xd66e}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc016ac1280, {0xc00640a000, 0xd3e6, 0xc01d4e6140}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc018c0f588, {0xc00640a000, 0x7eda79, 0xc00640a28d}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01d153e00, {0xc00640a000, 0xb, 0xd66e}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2691d40, {0xc00640a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc01346c5f8, {0x631b900, 0xc0b2691d40}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc01346c380, {0x7fe42ab59310, 0xc01d153e00}, 0xd3e6) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc01346c380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc01346c380, {0xc01daec000, 0x8000, 0x132b6010001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc01d4cdc80, {0xc00f400900, 0x9, 0xc0b2691d28}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc01d4cdc80}, {0xc00f400900, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00f400900, 0x9, 0xc000118060}, {0x631b600, 0xc01d4cdc80}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00f4008c0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc01347c1a0, 0x10bed2f, 0x63c85b8) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 669 [runnable]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0180e22d0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc01c94c120) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 670 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0180c6340) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 671 [runnable]: internal/poll.runtime_pollWait(0x7fe42ab97d60, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc01538b000, 0xc004e14000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc01538b000, {0xc004e14000, 0xe2f8, 0xe2f8}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc01538b000, {0xc004e14000, 0xd458, 0xc0178e9e00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc016f3f390, {0xc004e14000, 0x7eda79, 0xc004e14ea5}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0180d8420, {0xc004e14000, 0x7fe422ad19b0, 0xe2f8}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc044e34c00, {0xc004e14000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00e0573f8, {0x631b900, 0xc044e34c00}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00e057180, {0x7fe42ab59310, 0xc0180d8420}, 0xd458) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00e057180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00e057180, {0xc01dfba000, 0x8000, 0x100005353000001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc01df9e2a0, {0xc019575380, 0x9, 0xc0385def60}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc01df9e2a0}, {0xc019575380, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc019575380, 0x9, 0x53}, {0x631b600, 0xc01df9e2a0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc019575340) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0180c6340, 0x10bed2f, 0xc00cc21500) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc0180c6340}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 29520 [select, 54 minutes]: github.com/cockroachdb/cockroach/pkg/server/diagnostics.(*Reporter).PeriodicallyReportDiagnostics.func1({0x63c85b8, 0xc02b3c3ef0}) github.com/cockroachdb/cockroach/pkg/server/diagnostics/reporter.go:118 +0x2e5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802327 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030040800, {0x63c85b8, 0xc076c7da10}, 0xc0487bc500, 0xc04d955b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030040800, {0x63c85b8, 0xc076c7da10}, 0xc0487bc500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102174b2e5, 0x0, 0x0}, 0xc024e29470, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102174b2e5, 0x0, 0x0}, 0xc024e29470, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076c7d980}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076c7d980}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076c7d980}, {0xc00102e4e0}, 0xc0487bc400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076c7d920}, 0xc0487bc400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc02e4ea780}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc076c7d890}, {0xc00dfd36b0, 0x24}, 0x46c7c1446, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc044dedef0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc076c7d890}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800375 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc08fd52548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc08fd525b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07c5f76f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0541be700, {0xc0d8c14850, 0x2b, 0x61}, {0xc02cdd1b60, 0x2b, 0x60}, 0x1, 0x76) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07c5f7588, {0xc0d8c14850, 0x1, 0x60}, {0xc02cdd1b60, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07c5f7588, {0xc0d8c14850, 0x2b, 0x61}, {0xc02cdd1b60, 0x2b, 0x60}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07c5f6cc0, {0xc02cdd1b60, 0x60, 0x61}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07c5f6cc0, {0xc0d8c14850, 0x2b, 0xc02cdd1b60}, {0xc02cdd1b60, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07c5f6a00, {0xc02cdd1b60, 0x2b, 0x60}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc03decc008, {{0xc019cd8780, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0416d6100, {{0xc019cd8780, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc073e46800, {0x63c85b8, 0xc01d02c420}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01d02c420}, {0x64a6560, 0xc0416d6100}, {0xc019cd8780, 0x6e6f697372655670, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01d02c420}, {0x6466ed8, 0xc03decc000}, {0xc019cd8780, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0672e9668}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06adf9400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ab8c00, {0x63c85b8, 0xc01d02c420}, 0xc06adf9400, 0xc050b1b2c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ab8c00, {0x63c85b8, 0xc01d02c420}, 0xc06adf9400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01d02c390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d02c390}, {0xc0d68328f0, 0x660089}, 0xc0d68328f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01d02c390}, {0xc00102e4e0}, 0xc06adf9300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01d02c330}, 0xc06adf9300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b590}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 1133 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00cf4f380) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 1134 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab977c0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc01e395a80, 0xc02a064000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc01e395a80, {0xc02a064000, 0xcc53, 0xcc53}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc01e395a80, {0xc02a064000, 0xc375, 0xc020d48f00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc020d687e0, {0xc02a064000, 0x7eda79, 0xc02a0648e3}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0208f5c80, {0xc02a064000, 0x7fe13df71700, 0xcc53}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03c545da0, {0xc02a064000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00cf14cf8, {0x631b900, 0xc03c545da0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00cf14a80, {0x7fe42ab59310, 0xc0208f5c80}, 0xc375) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00cf14a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00cf14a80, {0xc02126a000, 0x8000, 0x100000556000001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc02124a060, {0xc00cb6c9e0, 0x9, 0xc03de96d00}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc02124a060}, {0xc00cb6c9e0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00cb6c9e0, 0x9, 0x0}, {0x631b600, 0xc02124a060}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00cb6c9a0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00cf4f380, 0x0, 0x104b6c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc00cf4f380}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 1036 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0198ce9c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 1037 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab976d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc020c89080, 0xc02fe00000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc020c89080, {0xc02fe00000, 0xcc53, 0xcc53}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc020c89080, {0xc02fe00000, 0xc540, 0xc021615040}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc021796728, {0xc02fe00000, 0x7eda79, 0xc02fe00718}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0217907e0, {0xc02fe00000, 0x7fe137d8db00, 0xcc53}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0342fd788, {0xc02fe00000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc01346c978, {0x631b900, 0xc0342fd788}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc01346c700, {0x7fe42ab59310, 0xc0217907e0}, 0xc540) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc01346c700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc01346c700, {0xc021968000, 0x8000, 0x100006d54000001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc021960360, {0xc02195c120, 0x9, 0xc0342fd770}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc021960360}, {0xc02195c120, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc02195c120, 0x9, 0x0}, {0x631b600, 0xc021960360}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc02195c0e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0198ce9c0, 0xc17dc161, 0x9b97ce0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc0198ce9c0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 219087612 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc073866090, 0x36) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc09176f500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc09176fb60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc050503300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc050503300, {0x63c8510, 0xc0738660c0}, 0xc054497dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01e4dfbc0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0738660c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01e4dfbc0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812607 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc052a11180, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc041065198, 0xc07d466438, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc07d466550, 0x0, 0xc07d466550) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc07d4663e8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d4663e8, {0xc05f79d720, 0x18, 0x94}, {0xc077407dd0, 0x18, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d4651c0, {0xc077407dd0, 0x0, 0xc0410654e0}, 0x121ea30, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d4651c0, {0xc05f79d720, 0x18, 0x30}, {0xc077407dd0, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d464f00, {0xc077407dd0, 0x18, 0x30}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0882f1b88, {{0xc08467a888, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0579e9600, {{0xc08467a888, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0562b0800, {0x63c85b8, 0xc04f63b1d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04f63b1d0}, {0x64a6560, 0xc0579e9600}, {0xc08467a888, 0x503000000, 0x18}, {0x1743011093dc9f96, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04f63b1d0}, {0x6466ed8, 0xc0882f1b80}, {0xc08467a888, 0x17, 0x18}, {0x1743011093dc9f96, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01e0694b8}, {{0x1743011093dc9f96, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011093dc9f96, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0455a2a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b1aa0496, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc04f63b1d0}, 0xc0455a2a00, 0xc046126dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc04f63b1d0}, 0xc0455a2a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011093dc9f96, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011093dc9f96, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63b140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b140}, {0xc04106b3f8, 0x660089}, 0xc04106b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63b140}, {0x4150121838f0f843}, 0xc0455a2800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63b0b0}, 0xc0455a2800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}, 0xc03ebdd420, 0xc031cfaf60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}, 0x3, 0xc033d33680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04106b980}}}, {0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}, 0xc03ebdd420, 0xc033d33680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63b0b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b0b0}, {0x40, 0xc04106bb70}, 0xc04106bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63b0b0}, {0x4e9d7a0, 0xc0455a2800}, 0xc03ebdd420, 0xc031cfaf60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63b0b0}, 0xc039f0a1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e0659e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e0659e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809082 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc062c31b30}, 0xc0681a33c8, 0x1, 0xc0264590a8, 0x519bbc, 0xc051a80600, 0xc03bfe0060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc062c31b30}, 0x1e0, 0xc026459230, 0x51a805a0, 0x0, 0x660069, 0xc051a80600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc062c31b30}, 0xc051a805a0, {{{{0x0, 0x0}, {0xc056f44240, 0x12a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc026459448, {0x63c85b8, 0xc062c31b30}, 0x20301c, 0x20301c) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc062c31b30}, {0xc072940c60, {0x1743011062a2d027, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc062c31b30}, 0xc01cc9b220, 0xc0264599f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc062c31b30}, 0x660089, {0xc072940c60, {0x1743011062a2d027, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc062c31b30}, 0xc051eb1b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011062a2d027, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011062a2d027, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046e8d1a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046e8d1a0}, {0xc02645b3f8, 0x660089}, 0xc02645b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046e8d1a0}, {0xc0015221e0}, 0xc077e16300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046e8d110}, 0xc077e16300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}, 0xc03e8c6420, 0xc013fc0d80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}, 0x3, 0xc054ffccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02645b980}}}, {0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}, 0xc03e8c6420, 0xc054ffccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046e8d110}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046e8d110}, {0x40, 0xc02645bb70}, 0xc02645bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046e8d110}, {0x4e9d7a0, 0xc077e16300}, 0xc03e8c6420, 0xc013fc0d80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046e8d110}, 0xc04e1b2060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc072940b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc072940b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 2613 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc012856880) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 2311 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1({0xc00d523200, 0xc0110c0f80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:203 +0x3b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2392 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1({0x63c85b8, 0xc018d32480}) github.com/cockroachdb/cockroach/pkg/server/node.go:747 +0x117 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2514 [select, 31 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7bc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:823 +0x11e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2312 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc09a9538c0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc09a9538c0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc09a9538c0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f4d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2313 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0646c29a0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0646c29a0, 0x19) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0646c29a0, 0xc021210be0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02f6ea118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02f6ea118, {0x63c85b8, 0xc02f6e6210}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc0046b9258, {0x63c85b8, 0xc02f6e6210}, {0x63fdb00, 0xc02f6ea330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc0046b9258, {0x63c85b8, 0xc02f6e6210}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2314 [semacquire]: sync.runtime_Semacquire(0x792f9ffa98) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc0ae8f37a0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0ae8f37a0, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0ae8f37a0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x7, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f560}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2315 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc08bd84360, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc08bd84360, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc08bd84360, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x38, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f590}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2316 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0184a7b00, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0184a7b00, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0184a7b00, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x51, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f5c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2317 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc062687c20, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc062687c20, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc062687c20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xe, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f5f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2318 [runnable]: sync.runtime_Semacquire(0x792f9ff1d4) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc051dcdc20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc051dcdc20, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc051dcdc20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x46, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f620}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2319 [semacquire]: sync.runtime_Semacquire(0x792f9ffba5) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc06280e120) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc06280e120, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc06280e120, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x32, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f650}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2320 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc01c29c5a0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc01c29c5a0, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc01c29c5a0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x8a, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f680}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2321 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc07561b9e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07561b9e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07561b9e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc052b3b470, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0xc00df9fc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f6b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2402 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc04a44be60, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04a44be60, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04a44be60, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x61, 0x5) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f920}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2403 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc07e07de40, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07e07de40, 0x54) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07e07de40, 0xc0133f4400) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc000dec918, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc000dec918, {0x63c85b8, 0xc013d681e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc08e159258, {0x63c85b8, 0xc013d681e0}, {0x63fdb00, 0xc000decb30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc08e159258, {0x63c85b8, 0xc013d681e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f950}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2404 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0923c25a0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0923c25a0, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0923c25a0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x24, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f980}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2405 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0326fa5a0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0326fa5a0, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0326fa5a0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x13b, 0x35) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f9b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2406 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc09a35a000, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc09a35a000, 0x70) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc09a35a000, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x40, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9f9e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2407 [semacquire]: sync.runtime_Semacquire(0x792f9ffb4f) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc0434a7200) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0434a7200, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0434a7200, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x55, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fa10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2408 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc086652c60, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc086652c60, 0x22) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc086652c60, 0xc02b6a10a0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02b6e3118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02b6e3118, {0x63c85b8, 0xc02b6f0600}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc01baf7258, {0x63c85b8, 0xc02b6f0600}, {0x63fdb00, 0xc02b6e3330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc01baf7258, {0x63c85b8, 0xc02b6f0600}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fa70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2409 [semacquire]: sync.runtime_Semacquire(0x792f9ffaa5) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc0416bb320) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0416bb320, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0416bb320, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x59, 0x3) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9faa0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2410 [semacquire]: sync.runtime_Semacquire(0x792f9ffb28) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc07561a240) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07561a240, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07561a240, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x1e, 0x3) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fad0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2411 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc06d558f20, 0x2) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc06d558f20, 0x11) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc06d558f20, 0xc025cedc00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc025db8d18, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc025db8d18, {0x63c85b8, 0xc025db49f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07c2e3258, {0x63c85b8, 0xc025db49f0}, {0x63fdb00, 0xc025db8f30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07c2e3258, {0x63c85b8, 0xc025db49f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fb00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2412 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc05f438b40, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc05f438b40, 0x70) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc05f438b40, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x4c, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fb30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2413 [semacquire]: sync.runtime_Semacquire(0x792f9ffb0f) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc059603d40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc059603d40, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc059603d40, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x38, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fb60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2414 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc04e86dc20, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04e86dc20, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04e86dc20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x70, 0x4) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fb90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2415 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc04fd38360, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04fd38360, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04fd38360, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x2f, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fbc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2416 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc07561bc20, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07561bc20, 0x10) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07561bc20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x1d, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fe30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2417 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc07e07d600, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07e07d600, 0x58) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07e07d600, 0xc021a88360) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc021ab8118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc021ab8118, {0x63c85b8, 0xc021aa8510}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07c477258, {0x63c85b8, 0xc021aa8510}, {0x63fdb00, 0xc021ab8330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07c477258, {0x63c85b8, 0xc021aa8510}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fe60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2418 [runnable]: sync.runtime_Semacquire(0x792f9ff116) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc0604a79e0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0604a79e0, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0604a79e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x2f, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fe90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2419 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0462b8360, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0462b8360, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0462b8360, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x16, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fec0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2420 [runnable]: sync.runtime_Semacquire(0x792f9ff266) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc00e987200) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc00e987200, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc00e987200, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x27, 0x3) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9fef0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2421 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0895f4dc0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0895f4dc0, 0x3f) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0895f4dc0, 0xc01d4d5740) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc01e149518, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc01e149518, {0x63c85b8, 0xc01e166f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07c5d3258, {0x63c85b8, 0xc01e166f60}, {0x63fdb00, 0xc01e149730}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07c5d3258, {0x63c85b8, 0xc01e166f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9ff20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2422 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0120d8240, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0120d8240, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0120d8240, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x6, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9ff80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2423 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc07561ac60, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07561ac60, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07561ac60, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x1f, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018c9ffb0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2424 [semacquire]: sync.runtime_Semacquire(0x792f9ffb33) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc04a04db00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04a04db00, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04a04db00, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x61, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2425 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc07e07d1e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07e07d1e0, 0x54) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07e07d1e0, 0xc018514a20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc018573118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc018573118, {0x63c85b8, 0xc0185927b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07c605258, {0x63c85b8, 0xc0185927b0}, {0x63fdb00, 0xc018573330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07c605258, {0x63c85b8, 0xc0185927b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4030}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2426 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0622c0fc0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0622c0fc0, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0622c0fc0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x71, 0x7) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4060}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2427 [semacquire]: sync.runtime_Semacquire(0x792f9ffaa2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc05b8dd440) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc05b8dd440, 0x50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc05b8dd440, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x34, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4090}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2428 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0862ee160, 0x2) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0862ee160, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0862ee160, 0xc025cecdc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc025d2a918, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc025d2a918, {0x63c85b8, 0xc025d3e960}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc005ad7258, {0x63c85b8, 0xc025d3e960}, {0x63fdb00, 0xc025d2ab30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc005ad7258, {0x63c85b8, 0xc025d3e960}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce40c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2429 [semacquire]: sync.runtime_Semacquire(0x792f9ffbaa) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc0609a7c20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0609a7c20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0609a7c20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc0443438f0, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0xc00df9fc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce40f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2430 [runnable]: sync.runtime_Semacquire(0x792f9ff112) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc05f8059e0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc05f8059e0, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc05f8059e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x21, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4360}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2431 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0bc19d8c0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0bc19d8c0, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0bc19d8c0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x38, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4390}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2432 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc014c06840, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc014c06840, 0x28) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc014c06840, 0xc02e9812e0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02e9e7118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02e9e7118, {0x63c85b8, 0xc02e9e18f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07c80b258, {0x63c85b8, 0xc02e9e18f0}, {0x63fdb00, 0xc02e9e7330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07c80b258, {0x63c85b8, 0xc02e9e18f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce43c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2433 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0524ff9e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0524ff9e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0524ff9e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02ce14f60, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0xc00df9fc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce43f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2434 [semacquire]: sync.runtime_Semacquire(0x792f9ffaf7) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc04d073560) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04d073560, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04d073560, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc044342720, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0xc00df9fc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4420}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2435 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc08535cdc0, 0x2) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc08535cdc0, 0x27) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc08535cdc0, 0xc021d45e80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc021de1918, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc021de1918, {0x63c85b8, 0xc021dbdfb0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07c907258, {0x63c85b8, 0xc021dbdfb0}, {0x63fdb00, 0xc021de1b30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07c907258, {0x63c85b8, 0xc021dbdfb0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4450}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2436 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc04b5f0fc0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04b5f0fc0, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04b5f0fc0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x6f, 0x7) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce44b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2437 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc05f438d80, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc05f438d80, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc05f438d80, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x1f, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce44e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2438 [semacquire]: sync.runtime_Semacquire(0x792f9ffa94) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc084511d40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc084511d40, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc084511d40, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x49, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4510}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2439 [semacquire]: sync.runtime_Semacquire(0x792f9ffac1) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc05749f680) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc05749f680, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc05749f680, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc0cec0d980, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4540}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2440 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0862ee000, 0x2) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0862ee000, 0x39) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0862ee000, 0xc02a303780) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02a365118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02a365118, {0x63c85b8, 0xc02a3783c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07cba3258, {0x63c85b8, 0xc02a3783c0}, {0x63fdb00, 0xc02a365330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07cba3258, {0x63c85b8, 0xc02a3783c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4570}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2441 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0609a79e0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0609a79e0, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0609a79e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x7, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce45a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2442 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc08c4d6480, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc08c4d6480, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc08c4d6480, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x18, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce45d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2443 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc01c0e06e0, 0x2) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc01c0e06e0, 0x2a) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc01c0e06e0, 0xc01f996d80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc01fa88918, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc01fa88918, {0x63c85b8, 0xc01fa92d20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc00d0cb258, {0x63c85b8, 0xc01fa92d20}, {0x63fdb00, 0xc01fa88b30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc00d0cb258, {0x63c85b8, 0xc01fa92d20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4600}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2444 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc08535c9a0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc08535c9a0, 0x6) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc08535c9a0, 0xc02d006ee0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02d0bd518, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02d0bd518, {0x63c85b8, 0xc02d0c5a70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07cdf3258, {0x63c85b8, 0xc02d0c5a70}, {0x63fdb00, 0xc02d0bd730}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07cdf3258, {0x63c85b8, 0xc02d0c5a70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4870}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2445 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc084af2b00, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc084af2b00, 0xe) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc084af2b00, 0xc023a83200) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc023bed918, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc023bed918, {0x63c85b8, 0xc023bc7c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc019be7258, {0x63c85b8, 0xc023bc7c50}, {0x63fdb00, 0xc023bedb30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc019be7258, {0x63c85b8, 0xc023bc7c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce48a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2446 [semacquire]: sync.runtime_Semacquire(0x792f9ffab4) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc08ec559e0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc08ec559e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc08ec559e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc044342330, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce48d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2447 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc037fe1e60, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc037fe1e60, 0x50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc037fe1e60, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x92, 0x6) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4900}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2448 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc039e87680, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc039e87680, 0x50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc039e87680, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x47, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4930}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2449 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc01cc76000, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc01cc76000, 0x5c) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc01cc76000, 0xc02005cb40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc020090918, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc020090918, {0x63c85b8, 0xc0200b2120}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc00f1bf258, {0x63c85b8, 0xc0200b2120}, {0x63fdb00, 0xc020090b30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc00f1bf258, {0x63c85b8, 0xc0200b2120}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4960}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2450 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc06c10aa20, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc06c10aa20, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc06c10aa20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x8d, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce49c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2451 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc01c0e18c0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc01c0e18c0, 0xa) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc01c0e18c0, 0xc0133f5760) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc000e35d18, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc000e35d18, {0x63c85b8, 0xc014060ba0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07cf29258, {0x63c85b8, 0xc014060ba0}, {0x63fdb00, 0xc000e35f30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07cf29258, {0x63c85b8, 0xc014060ba0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce49f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2452 [semacquire]: sync.runtime_Semacquire(0x792f9ffb44) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc04948d560) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04948d560, 0x20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04948d560, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x4c, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4a20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2453 [runnable]: sync.runtime_Semacquire(0x792f9ff11a) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc014d96fc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc014d96fc0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc014d96fc0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x5f, 0x4) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4a50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2454 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc04b38e7e0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04b38e7e0, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04b38e7e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x42, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4a80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2455 [semacquire]: sync.runtime_Semacquire(0x792f9ffaa0) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc04704aea0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04704aea0, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04704aea0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x18d, 0x40) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4ab0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2456 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0646c2420, 0x3) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0646c2420, 0x1f) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0646c2420, 0xc0214becc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc0215b8918, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc0215b8918, {0x63c85b8, 0xc0215c3920}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07d04f258, {0x63c85b8, 0xc0215c3920}, {0x63fdb00, 0xc0215b8b30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07d04f258, {0x63c85b8, 0xc0215c3920}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2457 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc079c18b40, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc079c18b40, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc079c18b40, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x38, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4b10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2458 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc05c0a3e60, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc05c0a3e60, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc05c0a3e60, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02bcb5320, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4d80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2459 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc01c0e1340, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc01c0e1340, 0x55) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc01c0e1340, 0xc0207bcd00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc0205ce918, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc0205ce918, {0x63c85b8, 0xc020912870}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc01fb1b258, {0x63c85b8, 0xc020912870}, {0x63fdb00, 0xc0205ceb30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc01fb1b258, {0x63c85b8, 0xc020912870}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4db0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2460 [semacquire]: sync.runtime_Semacquire(0x792f9ffb1a) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc09a35a240) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc09a35a240, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc09a35a240, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc065313dd0, 0x3) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4de0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2461 [runnable]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc01e506000, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc01e506000, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc01e506000, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x7c, 0x5) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4e10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2462 [semacquire]: sync.runtime_Semacquire(0x792f9ffbac) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc07bdf8fc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07bdf8fc0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07bdf8fc0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x4a, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4e40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2463 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0736e25a0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0736e25a0, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0736e25a0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x32, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4e70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2464 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc058088ea0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc058088ea0, 0x50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc058088ea0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x25, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4ed0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2465 [runnable]: sync.runtime_Semacquire(0x792f9ff3cf) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc01222db00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc01222db00, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc01222db00, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xf, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4f00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2466 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc014c06000, 0x2) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc014c06000, 0x46) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc014c06000, 0xc02fbb5560) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02fc3f518, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02fc3f518, {0x63c85b8, 0xc02fc4b5c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc013ee1258, {0x63c85b8, 0xc02fc4b5c0}, {0x63fdb00, 0xc02fc3f730}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc013ee1258, {0x63c85b8, 0xc02fc4b5c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4f30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2467 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc05118fe40, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc05118fe40, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc05118fe40, 0xc023a83100) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc023bec118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc023bec118, {0x63c85b8, 0xc023bc7230}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07e087258, {0x63c85b8, 0xc023bc7230}, {0x63fdb00, 0xc023bec330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07e087258, {0x63c85b8, 0xc023bc7230}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2468 [semacquire]: sync.runtime_Semacquire(0x792f9ffb90) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc074159c20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc074159c20, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc074159c20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x23, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4f90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2469 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0646c3ce0, 0x2) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0646c3ce0, 0x71) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0646c3ce0, 0xc024e27160) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc024e7b518, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc024e7b518, {0x63c85b8, 0xc024e7de90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc07e215258, {0x63c85b8, 0xc024e7de90}, {0x63fdb00, 0xc024e7b730}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc07e215258, {0x63c85b8, 0xc024e7de90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4fc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2470 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc086652dc0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc086652dc0, 0x1e) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc086652dc0, 0xc02bd561a0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02bd5e118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02bd5e118, {0x63c85b8, 0xc02bd50c30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc00d18f258, {0x63c85b8, 0xc02bd50c30}, {0x63fdb00, 0xc02bd5e330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc00d18f258, {0x63c85b8, 0xc02bd50c30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce4ff0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2471 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc04a04c7e0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc04a04c7e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc04a04c7e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xe6, 0x1a) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2472 [semacquire]: sync.runtime_Semacquire(0x792f9ffb13) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc066b670e0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc066b670e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc066b670e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc044342960, 0x1) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5290}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2473 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc08535cf20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc08535cf20, 0x24) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc08535cf20, 0xc02eeda300) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02eed7118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02eed7118, {0x63c85b8, 0xc02eed1a70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc082015258, {0x63c85b8, 0xc02eed1a70}, {0x63fdb00, 0xc02eed7330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc082015258, {0x63c85b8, 0xc02eed1a70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce52c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2474 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0580890e0, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0580890e0, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0580890e0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x12, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce52f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2475 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc065059ce0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc065059ce0, 0x34) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc065059ce0, 0xc02ee28f40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02ee69518, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02ee69518, {0x63c85b8, 0xc02ee7a720}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc082069258, {0x63c85b8, 0xc02ee7a720}, {0x63fdb00, 0xc02ee69730}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc082069258, {0x63c85b8, 0xc02ee7a720}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5320}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2476 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc086aac2c0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc086aac2c0, 0x61) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc086aac2c0, 0xc021df8400) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc021de5518, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc021de5518, {0x63c85b8, 0xc021df5920}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc082065258, {0x63c85b8, 0xc021df5920}, {0x63fdb00, 0xc021de5730}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc082065258, {0x63c85b8, 0xc021df5920}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5350}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2477 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc065c03c20, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc065c03c20, 0x20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc065c03c20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x6, 0x2) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5380}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2478 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc0895f4f20, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc0895f4f20, 0x28) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc0895f4f20, 0xc023a82d00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc023ba4118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc023ba4118, {0x63c85b8, 0xc023b94990}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc010741258, {0x63c85b8, 0xc023b94990}, {0x63fdb00, 0xc023ba4330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc010741258, {0x63c85b8, 0xc023b94990}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce53e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2479 [semacquire]: sync.runtime_Semacquire(0x792f9ffac6) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc06ee9c900) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc06ee9c900, 0x10) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc06ee9c900, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x64, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5410}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2480 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc07e07d080, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc07e07d080, 0x12) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc07e07d080, 0xc02fef3360) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc02ff57d18, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc02ff57d18, {0x63c85b8, 0xc02ff5f020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc082255258, {0x63c85b8, 0xc02ff5f020}, {0x63fdb00, 0xc02ff57f30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc082255258, {0x63c85b8, 0xc02ff5f020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5440}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2481 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc03c43b680, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc03c43b680, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc03c43b680, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x97, 0x6) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5470}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2482 [runnable]: sync.runtime_Semacquire(0x792f9ff344) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x123c72f) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/pebble.(*commitPipeline).publish(0xc000a6d300, 0xc039ea05a0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:402 +0x85 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc039ea05a0, 0x10) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:266 +0xfe github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc039ea05a0, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x5c, 0x6) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce54a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2483 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc086e729a0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc086e729a0, 0x29) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc086e729a0, 0xc002e20480) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc000af7d18, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc000af7d18, {0x63c85b8, 0xc013128360}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc076bed258, {0x63c85b8, 0xc013128360}, {0x63fdb00, 0xc000af7f30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc076bed258, {0x63c85b8, 0xc013128360}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce54d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2484 [runnable]: sync.runtime_SemacquireMutex(0xc0a6e7a000, 0xc0, 0x649cdb8) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000cca218) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/cockroach/pkg/util/metric.(*Histogram).RecordValue(0xc000cca1c0, 0xedb7a8351) github.com/cockroachdb/cockroach/pkg/util/metric/pkg/util/metric/metric.go:247 +0x57 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:973 +0x1405 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2485 [runnable]: github.com/cockroachdb/errors/markers.Is({0x0, 0x0}, {0x631bca0, 0xc000a260a0}) github.com/cockroachdb/errors/markers/external/com_github_cockroachdb_errors/markers/markers.go:44 +0x2fb github.com/cockroachdb/errors.Is(...) github.com/cockroachdb/errors/external/com_github_cockroachdb_errors/markers_api.go:35 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).stepRaftGroup.func1(0xc01ffe3b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:535 +0x288 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).withRaftGroupLocked.func1(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1772 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).withRaftGroupLocked(0xc021fcdc00, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1773 +0x24f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).withRaftGroup(0xc021fcdc00, 0xc0, 0xedb7a8351) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1818 +0x92 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).stepRaftGroup(0x63c85b8, 0xc021fc7a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:512 +0x45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processRaftRequestWithReplica(0x3fc3333333333333, {0x63c85b8, 0xc021fc7a40}, 0xc021fcdc00, 0xc0854d6098) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:262 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processRequestQueue.func1({0xc082603d68, 0x63c85b8}, 0xc018ce5530) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:467 +0x38 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).withReplicaForRequest(0xc0082db9d8, {0x63c85b8, 0xc018ce5530}, 0xc0854d6098, 0xc082603e28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:221 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processRequestQueue(0xc0082db200, {0x63c85b8, 0xc018ce5530}, 0x18782) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:465 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce5530}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:296 +0x1d2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2486 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc071392f20, 0x3) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc071392f20, 0x33) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc071392f20, 0xc021d440c0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0xc021d4e118, 0xb8) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).ApplyToStateMachine(0xc021d4e118, {0x63c85b8, 0xc021d31b90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:938 +0x194 github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc01a779258, {0x63c85b8, 0xc021d31b90}, {0x63fdb00, 0xc021d4e330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:286 +0x19a github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc01a779258, {0x63c85b8, 0xc021d31b90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:247 +0x9a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1028 +0x1672 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce57a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2487 [semacquire]: sync.runtime_SemacquireMutex(0x45954a, 0xc0, 0xc0017ca310) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc000a6e330) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/pebble.(*commitPipeline).prepare(0xc000a6d300, 0xc050f06a20, 0xaf) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:366 +0xd7 github.com/cockroachdb/pebble.(*commitPipeline).Commit(0xc000a6d300, 0xc050f06a20, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/commit.go:253 +0x5e github.com/cockroachdb/pebble.(*DB).Apply(0xc0002a1680, 0xc050f06a20, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/db.go:746 +0x1d1 github.com/cockroachdb/pebble.(*Batch).Commit(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/batch.go:905 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).Commit(0x40, 0x0) github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:485 +0x3c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:966 +0x137c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:648 +0x187 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc0082db200, 0x63c85b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:512 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00df8dc70, {0x63c85b8, 0xc018ce57d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:308 +0x25d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2488 [semacquire, 716 minutes]: sync.runtime_Semacquire(0xc00c185bb0) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0x9b97ce0) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:236 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2489 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc0082db200, {0x9b9, 0x3d3d674e695a5759}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:618 +0x156 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2490 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc0082db200, {0x63c85b8, 0xc018ce5860}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:678 +0x107 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2491 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func3({0xc00c99c600, 0x61633739303463}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1978 +0xbf github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2492 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x63c85b8, 0xc018ce5950}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2152 +0x2d6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2493 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x63c85b8, 0xc018ce5980}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2152 +0x2d6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2494 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x63c85b8, 0xc018ce59b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2152 +0x2d6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2529 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99f08, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00a7de180, 0xc011932a80, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00a7de180, {0xc011932a80, 0x924, 0x924}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00a7de180, {0xc011932a80, 0xc011932a85, 0x62}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc02e24c6f8, {0xc011932a80, 0x6, 0xc01fac6788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc03bc443f0, {0xc011932a80, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc000189778, {0x631b900, 0xc03bc443f0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc000189500, {0x63290e0, 0xc02e24c6f8}, 0x2) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc000189500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc000189500, {0xc001b74000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc0171c16e0, {0xc0105243c0, 0x9, 0xc037d4e750}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc0171c16e0}, {0xc0105243c0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0105243c0, 0x9, 0xc037d4e690}, {0x631b600, 0xc0171c16e0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc010524380) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0119761e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 2496 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1({0x63c85b8, 0xc018ce5a10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2256 +0x2c5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2497 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1({0x63c85b8, 0xc018ce5a40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2307 +0x2b6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2498 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1({0x63c85b8, 0xc018ce5d40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:235 +0x1b3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2515 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7bf0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:823 +0x11e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2516 [select, 8 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7c20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:823 +0x11e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2517 [chan send]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x17e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2518 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7c80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:823 +0x11e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2519 [chan send]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7cb0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x17e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2520 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7ce0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:823 +0x11e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2521 [chan send, 2 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7d10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:842 +0x17e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2522 [select, 294 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x63c85b8, 0xc017ed7d40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:823 +0x11e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2523 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc00f4002a0, {0x63c85b8, 0xc017ed7d70}, {0x189f1e0, 0x4d918c0, 0x0}, 0xc01d657400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:224 +0x1fd github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1(0xc02e6e6000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:286 +0x4a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc00f726060, 0xc06ba8d840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:436 +0x1f2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1({0x63c85b8, 0xc017ed7d70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:284 +0x1dd github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2361 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc015ae3800, 0xc00a7da9c0, {0x643dbd8, 0xc013755a90}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc018410570}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2362 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc015ae3860, 0xc00a7daa00, {0x643dbd8, 0xc03047fd60}, 0x1) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc0184105d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2499 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 2500 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 2363 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 2364 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 2501 [select, 716 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00d54b0e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0105b0210, {0x4cfe6e0, 0xc00c465d40}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc00d54ac60, 0xc00a8165b0, 0xc00a816580) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00d54ac60, {0x4cfe6e0, 0xc00c465d40}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc00a8aff20, {0x4cfe6e0, 0xc00c465d40}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc013755a90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc013755a90}, 0xc00a7da9c0, 0xc000cb3540, {0x63c85b8, 0xc018de23f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc018de23f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2365 [select, 716 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00de385a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00d9902c0, {0x4cfe6e0, 0xc01f861f80}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc031225d40, 0xc0110295b0, 0xc011029580) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc031225d40, {0x4cfe6e0, 0xc01f861f80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc02dc18e60, {0x4cfe6e0, 0xc01f861f80}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc03047fd60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc03047fd60}, 0xc00a7daa00, 0xc000cb3540, {0x63c85b8, 0xc018410c60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc018410c60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2531 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00c582200) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 2344 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00c7a06c0) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 2533 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc018d627e0}, 0xc00ab6e0a0, {0xc00e275d10, 0x12}, 0xc0161b2120) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc018d627e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2393 [select, 6 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.KVAdmissionControllerImpl.SetTenantWeightProvider.func1() github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3798 +0xe7 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver.KVAdmissionControllerImpl.SetTenantWeightProvider github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3791 +0xff goroutine 2394 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1({0x63c85b8, 0xc018d326f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:712 +0x1de github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2395 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound(0xc00175e1e0, 0xc01f786700, 0xc014381f50, 0x51cdee8, 0xc0017cad80, 0x0) github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:193 +0x15b github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func3({0xc000d4e600, 0xc00f726900}) github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:268 +0x49 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2396 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2({0x63c85b8, 0xc018d32870}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:165 +0x33b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219153676 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run(0xc072e46160, {0x63c85b8, 0xc0263951a0}, 0xc000d3ab40, {0x643def0, 0xc079b7d270}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:342 +0x1cb github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Receiver).PushUpdates(0xc01876a6c0, {0x643def0, 0xc079b7d270}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:89 +0x134 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb._SideTransport_PushUpdates_Handler({0x4cd47a0, 0xc01876a6c0}, {0x6400fe0, 0xc00e452d00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:346 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}, 0xc0354c7200, 0x51c46f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}, 0x3, 0xc074a81780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x16f0987}}}, {0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}, 0xc0354c7200, 0xc074a81780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0263950b0}, {0x460ef4, 0xc0006ecbc8}, 0xc0006ecb70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}, 0xc0354c7200, 0xc074a81780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc026af25b0}, 0xc0354c7200, 0x51c46f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc0b00d4d00}, 0xc0770ede60, 0xc007b7b7a0, 0x9181540, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0b00d4d00}, 0xc0770ede60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807400 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024c56000, {0x63c85b8, 0xc0606c3d40}, 0xc08a41fa00, 0xc04fe5c140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024c56000, {0x63c85b8, 0xc0606c3d40}, 0xc08a41fa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011056f702a4, 0x0, 0x0}, 0xc0906906b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011056f702a4, 0x0, 0x0}, 0xc0906906b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0606c3cb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0606c3cb0}, {0xc0052293f8, 0x660089}, 0xc005229400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0606c3cb0}, {0xc0015221e0}, 0xc08a41f800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0606c3c20}, 0xc08a41f800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}, 0xc07b4d6e00, 0xc046cf5f80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}, 0x3, 0xc08872ad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005229980}}}, {0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}, 0xc07b4d6e00, 0xc08872ad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0606c3c20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0606c3c20}, {0x40, 0xc005229b70}, 0xc005229ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0606c3c20}, {0x4e9d7a0, 0xc08a41f800}, 0xc07b4d6e00, 0xc046cf5f80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0606c3c20}, 0xc0534dea20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081fd4fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081fd4fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 2398 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache(0xc007b1a420, {0x63c85b8, 0xc018d32de0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:175 +0x25f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812503 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x80, 0xc05b456dc0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0c9977180, 0xc08507e010) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc05b456dc0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0c9977180, {0xc01c9b0e70, 0x5d, 0x6d}, {0xc02e545c80, 0x5d, 0x5d}, 0x0, 0x85) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0933f8438, {0xc01c9b0e70, 0x1, 0x5d}, {0xc02e545c80, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0933f8438, {0xc01c9b0e70, 0x5d, 0x6d}, {0xc02e545c80, 0x5d, 0x5d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0933f76c0, {0xc02e545c80, 0x5d, 0x6d}, 0x5d, 0x1f) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0933f76c0, {0xc01c9b0e70, 0x5d, 0xc02e545c80}, {0xc02e545c80, 0x5c, 0xc0}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0933f7400, {0xc02e545c80, 0x5d, 0x5d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc087983228, {{0xc085152840, 0x5c, 0xc0}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0202bd900, {{0xc085152840, 0x5c, 0xc0}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x64a6560, 0xc0202bd900}, {{0xc085152840, 0x5c, 0xc0}, {0x0, 0x0, 0x0}}, 0x0, 0xc07c140fc0) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:856 +0xbf github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal({0x63c85b8, 0xc04ae6b230}, {0x7fe42e217bc0, 0xc034107540}, {0x64a6560, 0xc0202bd900}, 0xc020024000, {0xc085152840, 0x5c, 0xc0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1287 +0x13d github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter({0x63c85b8, 0xc04ae6b230}, {0x7fe42e217bc0, 0xc034107540}, {0x64a6560, 0xc0202bd900}, 0x0, {0xc085152840, 0x5c, 0xc0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1073 +0x1cc github.com/cockroachdb/cockroach/pkg/storage.mvccInitPutUsingIter({0x63c85b8, 0xc04ae6b230}, {0x7fe42e217a50, 0xc034107540}, {0x64a6560, 0xc0202bd900}, 0x4c, {0xc085152840, 0x5c, 0xc0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1849 +0x1a5 github.com/cockroachdb/cockroach/pkg/storage.MVCCInitPut({0x63c85b8, 0xc04ae6b230}, {0x7fe42e217a50, 0xc034107540}, 0xc052f3d490, {0xc085152840, 0x5c, 0xc0}, {0x1743011082f2a274, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1814 +0x26a github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.InitPut({_, _}, {_, _}, {{0x64d2498, 0xc024ee3800}, {{0x1743011082f2a274, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_init_put.go:39 +0x254 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011082f2a274, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:493 +0x287 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05707ec00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:644 +0x190 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:612 +0x2da github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:428 +0x5f3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc024ee3800, {0x63c85b8, 0xc04ae6b230}, {0xc06066c2d8, 0x8}, 0xc05707ec00, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:673 +0x1c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x0, {0x63c85b8, _}, {_, _}, _, {{{0x1742dad62db4debf, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:773 +0xaa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc024ee3800, {0x63c85b8, _}, _, _, {{{0x1742dad62db4debf, 0x0, 0x0}, 0x0, {0x1f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:113 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024ee3800, {0x63c85b8, 0xc04ae6b230}, 0xc05707ec00, 0xc048082c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:167 +0x7b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024ee3800, {0x63c85b8, 0xc04ae6b230}, 0xc05707ec00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ae6b1a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ae6b1a0}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ae6b1a0}, {0xc00102e4e0}, 0xc05707eb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ae6b140}, 0xc05707eb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc018086e70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2562 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00ae684b0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0171c1800) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 2347 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00c7a11c0) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 219805063 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc0461eecf0}, 0xc05b94f8c0, 0x1, 0x708017dbe735a3, 0x37e11d600, 0xc0575c96e0, 0xc05638fe60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0461eecf0}, 0x1e0, 0xc093e2d230, 0x575c9680, 0xc01339d1e0, 0x18, 0xc0575c96e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc0461eecf0}, 0xc0575c9680, {{{{0x0, 0x0}, {0xc056cb4fc0, 0x3a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0461eecf0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc0461eecf0}, {0xc072ce17a0, {0x174301103e261a1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc0461eecf0}, 0xc077bdaa00, 0xc093e2d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0461eecf0}, 0x660089, {0xc072ce17a0, {0x174301103e261a1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0461eecf0}, 0xc089022c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103e261a1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103e261a1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0461eec60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0461eec60}, {0xc093e2f3f8, 0x660089}, 0xc093e2f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0461eec60}, {0x4677c5}, 0xc089022a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0461eeb70}, 0xc089022a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}, 0xc01af61000, 0xc05b94f878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}, 0x3, 0xc0733634c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc093e2f980}}}, {0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}, 0xc01af61000, 0xc0733634c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0461eeb70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0461eeb70}, {0x40, 0xc093e2fb70}, 0xc093e2fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}, 0x20301c, 0x20301c) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0461eeb70}, {0x4e9d7a0, 0xc089022a00}, 0xc01af61000, 0xc05b94f878) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0461eeb70}, 0xc078c038c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc072ce1560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc072ce1560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28288 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1({0x63c8510, 0xc017d65400}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:116 +0x15a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 2541 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc015af39b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc00abd2980}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}, 0xc014b68198, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}, 0x3, 0xc00c582d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x0}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}, 0xc014b68198, 0xc00c582d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0190b3cb0}, {0x460ef4, 0xc031b22bc8}, 0xc031b22b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}, 0xc014b68198, 0xc00c582d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc01437cdd0}, 0xc014b68198, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc00cf4f380}, 0xc00d7fe6c0, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc00cf4f380}, 0xc00d7fe6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 2534 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc016ce8de0, 0xc0181fa480, {0x643dbd8, 0xc0153e2510}, 0x20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc018d62c90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2535 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc016ce8f00, 0xc0181fa4c0, {0x643dbd8, 0xc0160daba0}, 0xd8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc018d62f90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2536 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00c582600) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 2350 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 2538 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc018d63590}, 0xc00ab6e410, {0xc011896480, 0x11}, 0xc0161b2540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc018d63590}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219027356 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc06b04b290, {0x63c85b8, 0xc04ef9e720}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc017844600, {0x63c85b8, 0xc04ef9e720}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc017844600, {0x63c8510, 0xc022d7fa80}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc017844600, {0x63c8510, 0xc022d7fa80}, {0x63f7db8, 0x9bf6f48}, 0xc09132d088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc017844600, {0x63c85b8, 0xc082fa1050}, {{0x63f7db8, 0x9bf6f48}, {0xc09ef96457, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc017844600, {0x63c85b8, 0xc082fa1050}, {{0x63f7db8, 0x9bf6f48}, {0xc09ef96457, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc09ef96457, 0x6}, 0x0, 0x0}, {0x681f646, 0xedb7a834f, 0x0}, {0x681fe2c, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc017844600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc017844600, {0x63c8510, 0xc052cd5580}, 0xc0b99efdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc051f16ca0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc052cd5580}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc051f16ca0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 2346 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc018db3ef0}, 0xc00862df40, {0xc00e275d10, 0x12}, 0xc014bedb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc018db3ef0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2595 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001763860) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 2543 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00d4e23c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 2564 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:425 +0x1dc created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:398 +0x5b goroutine 2565 [select]: github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1() github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1261 +0xe7 created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider github.com/cockroachdb/cockroach/pkg/util/admission/granter.go:1256 +0x1a5 goroutine 2566 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1({0xc00c99d200, 0xc8c6010960cbe96c}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:789 +0x3b6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802325 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021a39c00, {0x63c85b8, 0xc04cc060f0}, 0xc0487bce00, 0xc0cc0b9180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021a39c00, {0x63c85b8, 0xc04cc060f0}, 0xc0487bce00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110217ea25d, 0x0, 0x0}, 0xc024e29b50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110217ea25d, 0x0, 0x0}, 0xc024e29b50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04cc06060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cc06060}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04cc06060}, {0xc00102e4e0}, 0xc0487bcd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04cc06000}, 0xc0487bcd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc02e4eb140}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc076c7d830}, {0xc00dfd36b0, 0x24}, 0x46c71d2a0, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc0899590e0, 0xc0199d5c78}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc076c7d830}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2552 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 2553 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 2554 [select, 716 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00e17e5a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0085f2000, {0x4cfe6e0, 0xc003628000}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc00e17e120, 0xc031b33db0, 0xc031b33d80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e17e120, {0x4cfe6e0, 0xc003628000}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc00b138480, {0x4cfe6e0, 0xc003628000}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc015db28a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc015db28a0}, 0xc0181fa000, 0xc000cb3540, {0x63c85b8, 0xc0190e2120}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc0190e2120}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800549 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc095f7e548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc095f7e5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc08fdf00f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc052aaca80, {0xc022a6f380, 0x2b, 0x72}, {0xc0ce559420, 0x2b, 0x6d}, 0x1, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc08fdeff88, {0xc022a6f380, 0x1, 0x6d}, {0xc0ce559420, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc08fdeff88, {0xc022a6f380, 0x2b, 0x72}, {0xc0ce559420, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc08fdef6c0, {0xc0ce559420, 0x6d, 0x72}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc08fdef6c0, {0xc022a6f380, 0x2b, 0xc0ce559420}, {0xc0ce559420, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc08fdef400, {0xc0ce559420, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05b616b08, {{0xc00a771a70, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc079be5e00, {{0xc00a771a70, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc051fcfc00, {0x63c85b8, 0xc0281df320}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0281df320}, {0x64a6560, 0xc079be5e00}, {0xc00a771a70, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0281df320}, {0x6466ed8, 0xc05b616b00}, {0xc00a771a70, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc05f7e2450}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03b840200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020af0400, {0x63c85b8, 0xc0281df320}, 0xc03b840200, 0xc078db37c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020af0400, {0x63c85b8, 0xc0281df320}, 0xc03b840200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0281df290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0281df290}, {0xc0030568f0, 0x660089}, 0xc0030568f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0281df290}, {0xc00102e4e0}, 0xc03ef21f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0281df200}, 0xc03ef21f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7bad0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2349 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc019176570}, 0xc00af3c500, {0xc011896e88, 0x12}, 0xc01889a240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc019176570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2580 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000b90000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 2542 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00ab6eb90, {0xc014b68190, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00ab6eb90, {0xc014b68190, 0xc00cfc62e8, 0xc0d6831a68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0190b3b60, {0xc014b68190, 0xc0d6831ae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc0190b3b60}, {0xc014b68190, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00d7fe6c0, {0xc014b68190, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc014b68180, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x0, 0xc00d7fe6c0, {0x0, 0x0}, 0x7fffffff, 0xc0d6831d90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0x0, {0x0, 0x0}, {0x4cfe5a0, 0xc038160438}, 0x1b0000001a, 0xc0d6831d90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc01437cdd0, {0x4cfe5a0, 0xc038160438}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc015af39b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc015af39b0}, 0xc00182acb0, {0x63c85b8, 0xc0191bc000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc0191bc000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2596 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99e18, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00111e580, 0xc0033f2000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00111e580, {0xc0033f2000, 0xcd62, 0xcd62}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00111e580, {0xc0033f2000, 0xc0033f2154, 0x32c}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000136ff0, {0xc0033f2000, 0x6, 0xc031b1e788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc017daf488, {0xc0033f2000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0008ab778, {0x631b900, 0xc017daf488}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0008ab500, {0x63290e0, 0xc000136ff0}, 0x2) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0008ab500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0008ab500, {0xc002bb6000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc0176c3920, {0xc0105b42e0, 0x9, 0xc0752089c0}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc0176c3920}, {0xc0105b42e0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0105b42e0, 0x9, 0xc075208900}, {0x631b600, 0xc0176c3920}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0105b42a0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001763860) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 2597 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00a8f9720, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0176c3a40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 2598 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 2599 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 2600 [select, 716 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00e278360) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc019558630, {0x4cfe6e0, 0xc00c0cf400}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc00ded5e60, 0xc010ffb5b0, 0xc010ffb580) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ded5e60, {0x4cfe6e0, 0xc00c0cf400}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc00a412d00, {0x4cfe6e0, 0xc00c0cf400}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc0160daba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc0160daba0}, 0xc0181fa4c0, 0xc000cb3540, {0x63c85b8, 0xc017d785a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc017d785a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2581 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99c38, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00a7de380, 0xc001d6c000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00a7de380, {0xc001d6c000, 0xcd65, 0xcd65}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00a7de380, {0xc001d6c000, 0xc001d6c005, 0x152}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01f856628, {0xc001d6c000, 0x6, 0xc001f16788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc05bec8bd0, {0xc001d6c000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc01346d078, {0x631b900, 0xc05bec8bd0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc01346ce00, {0x63290e0, 0xc01f856628}, 0x2) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc01346ce00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc01346ce00, {0xc002c7e000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc0175d2d80, {0xc02195c3c0, 0x9, 0xc047dfdad0}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc0175d2d80}, {0xc02195c3c0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc02195c3c0, 0x9, 0xc047dfda10}, {0x631b600, 0xc0175d2d80}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc02195c380) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000b90000) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 2601 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc0160dae90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc00a412da0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}, 0xc014e560c0, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}, 0x3, 0xc00c0cf540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x203000}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}, 0xc014e560c0, 0xc00c0cf540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc017d78840}, {0x460ef4, 0xc002ca3bc8}, 0xc002ca3b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}, 0xc014e560c0, 0xc00c0cf540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a000}, 0xc014e560c0, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00e2785a0, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00e2785a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 2582 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00acd29b0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0175d2f60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 2602 [runnable]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00a8f9b30, {0xc014e560b8, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00a8f9b30, {0xc014e560b8, 0xc01f841080, 0xc06131fa68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc017d78750, {0xc014e560b8, 0xc06131fae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc017d78750}, {0xc014e560b8, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e2785a0, {0xc014e560b8, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc014e560a8, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x58, 0xc00e2785a0, {0x0, 0x0}, 0x7fffffff, 0xc06131fd90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x63474f8, {0x7fe42ab594c0, 0xc000808f40}, 0x0, {0x0, 0x0}, {0x4cfe5a0, 0xc023bdffc8}, 0xc0082db200, 0xc06131fd90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00185a000, {0x4cfe5a0, 0xc023bdffc8}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0160dae90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc0160dae90}, 0xc00182b8b0, {0x63c85b8, 0xc017d78930}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc017d78930}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2544 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99d28, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00111e800, 0xc001d38000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00111e800, {0xc001d38000, 0x9d04, 0x9d04}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00111e800, {0xc001d38000, 0xc001d38005, 0x475}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000f06188, {0xc001d38000, 0x6, 0xc001f18788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc02b06aae0, {0xc001d38000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc001073e78, {0x631b900, 0xc02b06aae0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc001073c00, {0x63290e0, 0xc000f06188}, 0x2) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc001073c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc001073c00, {0xc002ca4000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc017711740, {0xc00cb6c3c0, 0x9, 0xc073ab3260}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc017711740}, {0xc00cb6c3c0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00cb6c3c0, 0x9, 0xc073ab31a0}, {0x631b600, 0xc017711740}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00cb6c380) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00d4e23c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 2545 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00ab6efa0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc017711920) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 2351 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 2352 [select, 716 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00e2c45a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0010493f0, {0x4cfe6e0, 0xc00c7a1880}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc00e2c4120, 0xc0199d55b0, 0xc0199d5580) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e2c4120, {0x4cfe6e0, 0xc00c7a1880}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc00aa77a80, {0x4cfe6e0, 0xc00c7a1880}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc0153e2510) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc0153e2510}, 0xc0181fa480, 0xc000cb3540, {0x63c85b8, 0xc019177da0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc019177da0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2603 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc0160db240}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc00a412ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}, 0xc014e562b8, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}, 0x3, 0xc00c0cf800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x78}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}, 0xc014e562b8, 0xc00c0cf800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc017d78db0}, {0x460ef4, 0xc001f12bc8}, 0xc001f12b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}, 0xc014e562b8, 0xc00c0cf800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a0d0}, 0xc014e562b8, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc00e278a20, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc00e278a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 2604 [runnable]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00a8f9c20, {0xc014e562b0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00a8f9c20, {0xc014e562b0, 0xc01f8410c8, 0xc005a1ba68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc017d78c90, {0xc014e562b0, 0xc005a1bae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc017d78c90}, {0xc014e562b0, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e278a20, {0xc014e562b0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc014e562a0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x58, 0xc00e278a20, {0x0, 0x0}, 0x7fffffff, 0xc005a1bd90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x63474f8, {0x7fe42ab594c0, 0xc000808f40}, 0x0, {0x0, 0x0}, {0x4cfe5a0, 0xc02f752d08}, 0xc0082db200, 0xc005a1bd90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00185a0d0, {0x4cfe5a0, 0xc02f752d08}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0160db240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc0160db240}, 0xc0018620b0, {0x63c85b8, 0xc017d78ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc017d78ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2244 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc002e51b18) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc0231ebf50) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0231ebf50, {0x63c8510, 0xc0036c5c00}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x63c85b8, 0xc018b701e0}, {0xc002e51c98, 0x2, 0x4000000000000000}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:193 +0x91 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001306580, {0x63c85b8, 0xc018b701e0}, 0xc0194c3fb0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:291 +0x5de github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x63c85b8, 0xc018b70120}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:215 +0x339 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28328 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe995a8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00aa31600, 0xc01439a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00aa31600, {0xc01439a000, 0xcd66, 0xcd66}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00aa31600, {0xc01439a000, 0xc01439a005, 0x22}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0304baf08, {0xc01439a000, 0x6, 0xc002d9d788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc02f753f50, {0xc01439a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00b3ae278, {0x631b900, 0xc02f753f50}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00b3ae000, {0x63290e0, 0xc0304baf08}, 0xcd66) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00b3ae000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00b3ae000, {0xc00450e000, 0x8000, 0x48d7e7}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc014b025a0, {0xc005a502e0, 0x9, 0xc002d9dd18}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc014b025a0}, {0xc005a502e0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc005a502e0, 0x9, 0xc002d9dd6f}, {0x631b600, 0xc014b025a0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc005a502a0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0003f41e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 2607 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc0160db820}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc00a413100}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}, 0xc014e56750, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}, 0x3, 0xc00c0cfdc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x63c8580}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}, 0xc014e56750, 0xc00c0cfdc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc017d79b60}, {0x460ef4, 0xc001f13bc8}, 0xc001f13b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}, 0xc014e56750, 0xc00c0cfdc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc00185a270}, 0xc014e56750, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00e279e60, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00e279e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 2608 [runnable]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00b0a82d0, {0xc014e56748, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00b0a82d0, {0xc014e56748, 0xc01f841140, 0xc0020b1a68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc017d79a40, {0xc014e56748, 0xc0020b1ae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc017d79a40}, {0xc014e56748, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e279e60, {0xc014e56748, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc014e56738, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x58, 0xc00e279e60, {0x0, 0x0}, 0x7fffffff, 0xc0020b1d90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x63474f8, {0x7fe42ab594c0, 0xc000808f40}, 0x0, {0x0, 0x0}, {0x4cfe5a0, 0xc023bdff80}, 0xc0082db200, 0xc0020b1d90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00185a270, {0x4cfe5a0, 0xc023bdff80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0160db820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc0160db820}, 0xc001429bb0, {0x63c85b8, 0xc017d79c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc017d79c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2609 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc017944240, 0xc019810dc0, {0x643dbd8, 0xc0160dbb00}, 0xd8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc018fa2960}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2610 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 2611 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 2612 [select, 716 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00e3967e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc019558840, {0x4cfe6e0, 0xc00e4e3ac0}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc00e396360, 0xc0199d5db0, 0xc0199d5d80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e396360, {0x4cfe6e0, 0xc00e4e3ac0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc00a413320, {0x4cfe6e0, 0xc00e4e3ac0}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc0160dbb00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc0160dbb00}, 0xc019810dc0, 0xc000cb3540, {0x63c85b8, 0xc018fa3290}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc018fa3290}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802523 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc07301fc50}, {0xc01c1471e0, 0x2, 0xc09c0c6e58}, {0x0, {0x1743010fee6d25b2, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc07301fc50}, 0xbefa, 0xc06f100900, 0x20, 0xc08308b0d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc07301fc50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809532 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc02b82c9c0}, 0xc075e1b0f8, 0x1, 0xc026b050a8, 0x519bbc, 0xc07054a2a0, 0xc06f2bea80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02b82c9c0}, 0x120, 0xc026b05230, 0x7054a240, 0x0, 0x660069, 0xc07054a2a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc02b82c9c0}, 0xc07054a240, {{{{0x0, 0x0}, {0xc05133e000, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02b82c9c0}, 0x6352410, 0x3102ee10) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc02b82c9c0}, {0xc07054a120, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc02b82c9c0}, 0xc02eeaa140, 0xc026b059f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02b82c9c0}, 0x660089, {0xc07054a120, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc02b82c9c0}, 0xc05b359600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b82c810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b82c810}, {0xc026b073f8, 0x660089}, 0xc026b07400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b82c810}, {0xc0015221e0}, 0xc05b359400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02b82c690}, 0xc05b359400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}, 0xc024415ac0, 0xc0ccf022d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}, 0x3, 0xc0af72d000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026b07980}}}, {0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}, 0xc024415ac0, 0xc0af72d000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02b82c690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b82c690}, {0x40, 0xc026b07b70}, 0xc026b07ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02b82c690}, {0x4e9d7a0, 0xc05b359400}, 0xc024415ac0, 0xc0ccf022d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02b82c690}, 0xc02f4cd680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07054a000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07054a000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805947 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc09108fc80}, 0xc02790bcb0, {0x646cc48, 0xc09108fce0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc09108fc80}, 0xedb7a834f, 0xc05067e7e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc09108fc80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03c904e70}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc02790bef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc03c904e70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2568 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func1({0x63c85b8, 0xc018af87e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:847 +0x108 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2569 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc01876a990, {0x63c8510, 0xc01348d940}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:319 +0x997 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x63c85b8, 0xc018af8810}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:189 +0x274 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2570 [select, 6 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).startSystemLogsGC.func1({0x63c85b8, 0xc018af8870}) github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:179 +0x169 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28621 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219801343 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc067188390}, 0xc0408bbce0, 0x1, 0xc01b18d0a8, 0x519bbc, 0xc034d0cba0, 0xc03278f8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc067188390}, 0x1e0, 0xc01b18d230, 0x34d0cb40, 0x0, 0x660069, 0xc034d0cba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc067188390}, 0xc034d0cb40, {{{{0x0, 0x0}, {0xc0748c4900, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc067188390}, 0x203013, 0x203013) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc067188390}, {0xc04e8c0120, {0x1743010ffb90bf72, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc067188390}, 0xc022383720, 0xc01b18d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067188390}, 0x660089, {0xc04e8c0120, {0x1743010ffb90bf72, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc067188390}, 0xc06308cb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffb90bf72, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffb90bf72, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0671882d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0671882d0}, {0xc01b18f3f8, 0x660089}, 0xc01b18f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0671882d0}, {0x57}, 0xc06308c900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067188240}, 0xc06308c900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}, 0xc0137dcba0, 0xc026e96780) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}, 0x3, 0xc08714fec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01b18f980}}}, {0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}, 0xc0137dcba0, 0xc08714fec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc067188240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067188240}, {0x40, 0xc01b18fb70}, 0xc01b18fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc067188240}, {0x4e9d7a0, 0xc06308c900}, 0xc0137dcba0, 0xc026e96780) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc067188240}, 0xc0450baea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc01c293440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc01c293440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 2615 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc01bba8360}, 0xc00b0a88c0, {0xc011896420, 0x12}, 0xc011723b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc01bba8360}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 2583 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000b90f00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 2584 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99b48, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc02f1ad080, 0xc0053a6000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02f1ad080, {0xc0053a6000, 0xe238, 0xe238}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc02f1ad080, {0xc0053a6000, 0xc0053a6005, 0x22f}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000800ab0, {0xc0053a6000, 0x6, 0xc031b24788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc0b2691a70, {0xc0053a6000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc001461af8, {0x631b900, 0xc0b2691a70}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc001461880, {0x63290e0, 0xc000800ab0}, 0x2) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc001461880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc001461880, {0xc002e22000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc017ca7aa0, {0xc02195c660, 0x9, 0xc086fc74d0}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc017ca7aa0}, {0xc02195c660, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc02195c660, 0x9, 0xc086fc7410}, {0x631b600, 0xc017ca7aa0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc02195c620) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000b90f00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 219802707 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc03003d800, {0x63c85b8, 0xc02ecddbf0}, 0xc0a7741900, 0xc075ed4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc03003d800, {0x63c85b8, 0xc02ecddbf0}, 0xc0a7741900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011000b213c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011000b213c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ecdd9b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ecdd9b0}, {0xc04e5713f8, 0x660089}, 0xc04e571400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ecdd9b0}, {0xc0015221e0}, 0xc0a7741700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ecdd410}, 0xc0a7741700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}, 0xc0b9209d00, 0xc015d8a5a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}, 0x3, 0xc07443d880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04e571980}}}, {0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}, 0xc0b9209d00, 0xc07443d880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02ecdd410}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ecdd410}, {0x40, 0xc04e571b70}, 0xc04e571ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02ecdd410}, {0x4e9d7a0, 0xc0a7741700}, 0xc0b9209d00, 0xc015d8a5a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02ecdd410}, 0xc038d24960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c489200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c489200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805002 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ad30800, {0x63c85b8, 0xc05397ff80}, 0xc05204fa00, 0xc02a3c7040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ad30800, {0x63c85b8, 0xc05397ff80}, 0xc05204fa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110026baf99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110026baf99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05397fef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05397fef0}, {0xc0936bf3f8, 0x660089}, 0xc0936bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05397fef0}, {0x20301b}, 0xc05204f800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05397fe60}, 0xc05204f800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}, 0xc06463a0e0, 0xc0206b9cb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}, 0x3, 0xc0158b7b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0936bf980}}}, {0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}, 0xc06463a0e0, 0xc0158b7b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05397fe60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05397fe60}, {0x40, 0xc0936bfb70}, 0xc0936bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}, 0x203005, 0x203005) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05397fe60}, {0x4e9d7a0, 0xc05204f800}, 0xc06463a0e0, 0xc0206b9cb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05397fe60}, 0xc033cc7620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc071535320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc071535320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 2585 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00acd2f00, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc017ca7bc0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 28232 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x63c85b8, 0xc02cc8b980}) github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:268 +0x32e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809807 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc07412aa50}, 0xc023f92d20, 0x1, 0xc005ebb0a8, 0x519bbc, 0xc05c89ad80, 0xc059b49980) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0d06db620, {0x63c85b8, 0xc07412aa50}, 0xc00e3e1c20, 0xc005ebb1a8, 0x5c89ad80, 0x0, 0x0, 0xc05c89ad80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc07412aa50}, 0xc02a80da40, {{{{0xc079db7b00, 0x46}, {0xc00449dd40, 0x82}}, {{0xc019560d80, 0x9c}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07412aa50}, 0x63521a0, 0x603b9ed8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc07412aa50}, {0xc07bd56360, {0x1743011003ba883b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc07412aa50}, 0xc066131ea0, 0xc005ebb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07412aa50}, 0x660089, {0xc07bd56360, {0x1743011003ba883b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07412aa50}, 0xc04d6abc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011003ba883b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011003ba883b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07412a9c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07412a9c0}, {0xc005ebd3f8, 0x660089}, 0xc005ebd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07412a9c0}, {0xc0015221e0}, 0xc04d6ab800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07412a900}, 0xc04d6ab800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}, 0xc032c16880, 0xc073de4630) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}, 0x3, 0xc062edebc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ebd980}}}, {0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}, 0xc032c16880, 0xc062edebc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07412a900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07412a900}, {0x40, 0xc005ebdb70}, 0xc005ebdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07412a900}, {0x4e9d7a0, 0xc04d6ab800}, 0xc032c16880, 0xc073de4630) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07412a900}, 0xc02a80d6e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04aa28ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04aa28ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810254 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01dd05000, {0x63c85b8, 0xc079483d40}, 0xc08467ff00, 0xc044f79d60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc079483d40}, 0xc08467ff00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cab7ef1, 0x0, 0x0}, 0xc04ac86aa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cab7ef1, 0x0, 0x0}, 0xc04ac86aa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079483cb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079483cb0}, {0xc026fdb3f8, 0x660089}, 0xc026fdb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079483cb0}, {0xc01b5bdae0}, 0xc08467fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079483c20}, 0xc08467fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}, 0xc01dcf2f20, 0xc0462f7c50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}, 0x3, 0xc07ba16400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026fdb980}}}, {0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}, 0xc01dcf2f20, 0xc07ba16400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079483c20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079483c20}, {0x40, 0xc026fdbb70}, 0xc026fdbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079483c20}, {0x4e9d7a0, 0xc08467fd00}, 0xc01dcf2f20, 0xc0462f7c50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079483c20}, 0xc01ceb2f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0616e3c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0616e3c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812792 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807998 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc06df377a0}, 0xc0974d8f60, 0x1, 0xc005b6d0a8, 0x519bbc, 0xc076f28600, 0xc03a90a3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc067aab020, {0x63c85b8, 0xc06df377a0}, 0xc073da77a0, 0xc005b6d1a8, 0x1, 0x0, 0x0, 0xc076f28600, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc06df377a0}, 0xc076f285a0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0321a8000, 0x68}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc06df377a0}, 0x63523b0, 0x2ad6efa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc06df377a0}, {0x0, {0x174301105a6b0f3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc06df377a0}, 0xc01664e0a0, 0xc005b6d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06df377a0}, 0x660089, {0x0, {0x174301105a6b0f3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc06df377a0}, 0xc0451e7900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a6b0f3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a6b0f3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301105a6b0f3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06df37710}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06df37710}, {0xc005b6f3f8, 0x660089}, 0xc005b6f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06df37710}, {0x0}, 0xc0451e7600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06df37680}, 0xc0451e7600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}, 0xc09d3b6920, 0xc053330960) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}, 0x3, 0xc0357978c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005b6f980}}}, {0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}, 0xc09d3b6920, 0xc0357978c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06df37680}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06df37680}, {0x40, 0xc005b6fb70}, 0xc005b6fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06df37680}, {0x4e9d7a0, 0xc0451e7600}, 0xc09d3b6920, 0xc053330960) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06df37680}, 0xc08ae0ff20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc076f28360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc076f28360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28161 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc00309dcf8) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01c226b40) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01c226b40, {0x63c8510, 0xc0203a8940}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00167cf00, {0x63c8510, 0xc01e61bac0}, {0xc02cf44c60, 0x1, 0xc002f46728}, {0xc007b7b5f0, 0xb970e0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:139 +0x625 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x0, {0x63c8510, 0xc01e61bac0}, {0xc02cf44c60, 0x63c85b8, 0xc01c879710}, {0x70777dc2cd10301, 0x63c85b8, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:78 +0x45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x63c8510, 0xc01e61bac0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:285 +0x7a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219804354 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ffd52c0, {0x63c85b8, 0xc04e453b30}, 0xc00beed5c0, 0x1, 0xc0501b70a8, 0x519bbc, 0xc057c03800, 0xc05866c180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04e453b30}, 0x120, 0xc0501b7230, 0x57c037a0, 0x0, 0x660069, 0xc057c03800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ffd52c0, {0x63c85b8, 0xc04e453b30}, 0xc057c037a0, {{{{0x0, 0x0}, {0xc09cc33320, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04e453b30}, 0x6352410, 0x44d53eb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01f9d9590, {0x63c85b8, 0xc04e453b30}, {0xc057c03680, {0x1743011032101887, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01f9d9590, {0x63c85b8, 0xc04e453b30}, 0xc050925040, 0xc0501b79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e453b30}, 0x660089, {0xc057c03680, {0x1743011032101887, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020010800, {0x63c85b8, 0xc04e453b30}, 0xc07b49cd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032101887, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032101887, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e453aa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e453aa0}, {0xc0501b93f8, 0x660089}, 0xc0501b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e453aa0}, {0x7fe32e6a42a0}, 0xc07b49cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e453a10}, 0xc07b49cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}, 0xc075dead80, 0xc0446bb410) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}, 0x3, 0xc06d644100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0501b9980}}}, {0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}, 0xc075dead80, 0xc06d644100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e453a10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e453a10}, {0x40, 0xc0501b9b70}, 0xc0501b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e453a10}, {0x4e9d7a0, 0xc07b49cb00}, 0xc075dead80, 0xc0446bb410) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e453a10}, 0xc03137c3c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057c03560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057c03560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806180 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc024953f20}, 0xc03fd03f08, 0x1, 0xc006efb0a8, 0x519bbc, 0xc0913c9e60, 0xc0925caf60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc024953f20}, 0x1e0, 0xc006efb230, 0x913c9e00, 0x0, 0x660069, 0xc0913c9e60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc024953f20}, 0xc0913c9e00, {{{{0x0, 0x0}, {0xc09cb78900, 0x5a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc024953f20}, 0x20301c, 0x20301c) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc024953f20}, {0xc0784b79e0, {0x174301104be44f22, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc024953f20}, 0xc0667463c0, 0xc006efb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024953f20}, 0x660089, {0xc0784b79e0, {0x174301104be44f22, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc024953f20}, 0xc06749eb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104be44f22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104be44f22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024953e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024953e90}, {0xc006efd3f8, 0x660089}, 0xc006efd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024953e90}, {0xc0015221e0}, 0xc06749e900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024953e00}, 0xc06749e900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}, 0xc076b10040, 0xc08aad8e70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}, 0x3, 0xc027088600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006efd980}}}, {0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}, 0xc076b10040, 0xc027088600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc024953e00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024953e00}, {0x40, 0xc006efdb70}, 0xc006efdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc024953e00}, {0x4e9d7a0, 0xc06749e900}, 0xc076b10040, 0xc08aad8e70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc024953e00}, 0xc01d9421e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04afc9b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04afc9b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806134 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc07279bda0}, {0xc070fa4700, 0x1, 0xc00aaafbb8}, {0x1, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntent(0x0, {0x63c85b8, 0xc07279bda0}, {{{0xc07b822c60, 0x22, 0x30}, {0x0, 0x0, 0x0}}, {{0x9, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:841 +0x9b github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc02fea22d0, {0x63c85b8, 0xc07279bda0}, {0xc0764d45a0, {0x1743011034ed316d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:549 +0x6db github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc07279bda0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc02fea22d0, {0x63c85b8, 0xc07279bda0}, {0xc0764d45a0, {0x1743011034ed316d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc07279bda0}, 0xc0ce9137c0, 0xc070fa5558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07279bda0}, 0xc0ce9137c0, {0xc0764d45a0, {0x1743011034ed316d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc07279bda0}, 0xc06fcccc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011034ed316d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011034ed316d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07279bd10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07279bd10}, {0xc006a6d3f8, 0x660089}, 0xc006a6d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07279bd10}, {0xd0300a280a200118}, 0xc06fccca00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07279bc80}, 0xc06fccca00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}, 0xc078eae0a0, 0xc08d7fa078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}, 0x3, 0xc06deba900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006a6d980}}}, {0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}, 0xc078eae0a0, 0xc06deba900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07279bc80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07279bc80}, {0x40, 0xc006a6db70}, 0xc006a6dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07279bc80}, {0x4e9d7a0, 0xc06fccca00}, 0xc078eae0a0, 0xc08d7fa078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07279bc80}, 0xc053192cc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0764d4360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0764d4360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219797015 [semacquire]: sync.runtime_SemacquireMutex(0xc00bd02b60, 0x3, 0xc00bd02b60) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc01bbf8338) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fe194b1a440, 0x7e61, 0x7fd5892e0200) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc02191a0c0, 0x7fe194b1a401, 0x7fe194b1a460, 0x4269, 0x426e) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc02190d080, {0x1db5a3c, 0x4269}, 0x0, 0xc04f285d78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04f285b00, {0xffffffffffffff15, 0xc01bbf85c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc04f285b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc04f285b00, {0xc044247d50, 0x2b, 0x6d}, 0x13, 0x6d) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04f285b00, {0xc04da69360, 0x2b, 0x6d}, {0xc044247d50, 0x51c9860, 0xc044247d50}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04f285b00, {0xc04da69360, 0x2b, 0x4d}, {0xc044247d50, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09171e398, {0xc04da69360, 0x2b, 0x4d}, {0xc044247d50, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09171ccc0, {0xc044247d50, 0x6d, 0x4d}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09171ccc0, {0xc04da69360, 0x2b, 0xc044247d50}, {0xc044247d50, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09171ca00, {0xc044247d50, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc062f94008, {{0xc00a7707e0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc082139e00, {{0xc00a7707e0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc051d35400, {0x63c85b8, 0xc048ddf1a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc048ddf1a0}, {0x64a6560, 0xc082139e00}, {0xc00a7707e0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc048ddf1a0}, {0x6466ed8, 0xc062f94000}, {0xc00a7707e0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04621a498}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc09cf11b00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020462400, {0x63c85b8, 0xc048ddf1a0}, 0xc09cf11b00, 0xc053b574a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020462400, {0x63c85b8, 0xc048ddf1a0}, 0xc09cf11b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048ddf110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048ddf110}, {0xc0936588f0, 0x660089}, 0xc0936588f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048ddf110}, {0xc00102e4e0}, 0xc09cf11a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048ddf0b0}, 0xc09cf11a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc058b21a70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811073 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc072608b70}, 0xc044154900, 0x1, 0xc0289230a8, 0x519bbc, 0xc016157da0, 0xc05770dda0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc072608b70}, 0x120, 0xc028923230, 0x16157d40, 0x0, 0x660069, 0xc016157da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc072608b70}, 0xc016157d40, {{{{0x0, 0x0}, {0xc04d19e900, 0x106}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc072608b70}, 0x6352410, 0x202c44a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc072608b70}, {0xc016157b00, {0x1743011073be7157, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc072608b70}, 0xc0a6e99f40, 0xc0289239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc072608b70}, 0x660089, {0xc016157b00, {0x1743011073be7157, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc072608b70}, 0xc04de36b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011073be7157, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011073be7157, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072608ae0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072608ae0}, {0xc0289253f8, 0x660089}, 0xc028925400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072608ae0}, {0x0}, 0xc04de36900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072608a50}, 0xc04de36900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}, 0xc0148def40, 0xc03a289bf0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}, 0x3, 0xc074716140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028925980}}}, {0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}, 0xc0148def40, 0xc074716140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc072608a50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072608a50}, {0x40, 0xc028925b70}, 0xc028925ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc072608a50}, {0x4e9d7a0, 0xc04de36900}, 0xc0148def40, 0xc03a289bf0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc072608a50}, 0xc03ffc06c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0161578c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0161578c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28613 [runnable]: sync.runtime_notifyListWait(0xc0003df278, 0xfc615) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xadd280) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc0003df260, {0x63c8510, 0xc00a7f5700}, 0x327c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xde github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x63c85b8, 0xc02cc73410}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x357 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219230298 [IO wait]: internal/poll.runtime_pollWait(0x7fdb124bd030, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0473a3580, 0xc0351d5000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0473a3580, {0xc0351d5000, 0x2b87, 0x2b87}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0473a3580, {0xc0351d5000, 0x2b87, 0xc04341fa40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc020d69fc8, {0xc0351d5000, 0x7eda79, 0xc0351d5005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc045af05a0, {0xc0351d5000, 0xc071eeec60, 0x2b87}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc047918480, {0xc0351d5000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc082745e78, {0x631b900, 0xc047918480}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc082745c00, {0x7fe42ab59310, 0xc045af05a0}, 0x2b87) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc082745c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc082745c00, {0xc070323000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03f82a1e0, {0xc070323000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc075c36b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc075c36b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc075c370e0, {0x634aac0, 0xc075c36b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc075c36a80, 0xc02cceb610, 0xc071eef42b, 0xc071eef42d, {0x635bec0, 0xc088e72360}, 0xc000d6ae00, 0xc09fbad318, 0xc071eef429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc075c36a80, {0x63c85b8, 0xc059dcb3b0}, 0xc02cceb600, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0402c8021, 0x8}}, 0x0, 0xc059dcb350, 0xc059dcb380, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc059dcb3b0}, {0x643d810, 0xc045af05a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06dc4f9b0}, {0x643d810, 0xc045af05a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06dc4f9b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802254 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0b2c00, {0x63c85b8, 0xc07130f380}, 0xc041b60300, 0xc0174088c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc07130f380}, 0xc041b60300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102260fe2d, 0x0, 0x0}, 0xc065ad0f70, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102260fe2d, 0x0, 0x0}, 0xc065ad0f70, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07130f2f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07130f2f0}, {0xc0768ef918, 0x660089}, 0xc0768ef920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07130f2f0}, {0xc00102e4e0}, 0xc041b60200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07130f290}, 0xc041b60200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06ea3ade0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07130f1d0}, {0xc00dfd36b0, 0x24}, 0x47ce6b26e, 0xc0387f9d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc06f0beab0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc07130f1d0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28151 [select]: github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal({{0x63c85b8, 0xc02c8fb5c0}, 0xc0198a6ba0, 0xc0198a6c00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:694 +0xbf github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:723 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc00167cf00, {0x63c8510, 0xc007b0fa80}, {{0xc01fffb778, 0x1, 0x8}, {0xc01fffb780, 0x1, 0x8}}, {0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:519 +0xd84 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc00167cf00, {0x63c8510, 0xc007b0fa80}, 0xc01d9f1a40, {{0xc01fffb778, 0x1, 0x8}, {0xc01fffb780, 0x1, 0x8}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:336 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1({0x63c8510, 0xc007b0fa80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:120 +0xbe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 28397 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc019955000, 0x6437410, 0xc00d963d40, {0x7fe42b743500, 0xc00b7e4d00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:254 +0x845 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:150 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc0082db200, 0xc00d963d40, {0x6437410, 0xc00b7e4d00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3082 +0xf7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc005a57f50, 0xc00d963d40, {0x6437410, 0xc00b7e4d00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:216 +0xf6 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x6437410, 0xc012f03b18, {0x6437410, 0xc00b7e4d00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1227 +0x38 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:753 +0xeb created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:750 +0x1cf goroutine 219801766 [semacquire]: sync.runtime_SemacquireMutex(0x2, 0x65, 0xcb769) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc00bd42340, 0x494a65, 0xc00bd42340) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x200000003, 0xc00bd42340, 0xc00009e800, 0xc0982cc440) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc025834b00, {0x40f5753, 0x35d3}, 0x0, 0xc060c77d78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc060c77b00, {0xffffffffffffff15, 0xc0982cc5c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc060c77b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc060c77b00, {0xc02bf99d80, 0x2b, 0x76}, 0x13, 0x5c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc060c77b00, {0xc03ed15420, 0x2b, 0x76}, {0xc02bf99d80, 0x51c9860, 0xc02bf99d80}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc060c77b00, {0xc03ed15420, 0x2b, 0xe0}, {0xc02bf99d80, 0x2b, 0x76}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc074ca4898, {0xc03ed15420, 0x2b, 0xe0}, {0xc02bf99d80, 0x2b, 0x76}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc074ca31c0, {0xc02bf99d80, 0x76, 0xe0}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc074ca31c0, {0xc03ed15420, 0x2b, 0xc02bf99d80}, {0xc02bf99d80, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc074ca2f00, {0xc02bf99d80, 0x2b, 0x76}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc060610108, {{0xc08b6c6b10, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0579e9d00, {{0xc08b6c6b10, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f262400, {0x63c85b8, 0xc050434300}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc050434300}, {0x64a6560, 0xc0579e9d00}, {0xc08b6c6b10, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc050434300}, {0x6466ed8, 0xc060610100}, {0xc08b6c6b10, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc015392a08}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc019e35800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020d8dc00, {0x63c85b8, 0xc050434300}, 0xc019e35800, 0xc0a6e99c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020d8dc00, {0x63c85b8, 0xc050434300}, 0xc019e35800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050434270}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050434270}, {0xc0936948f0, 0x660089}, 0xc0936948f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050434270}, {0xc00102e4e0}, 0xc019e35700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050434210}, 0xc019e35700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618cac90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812410 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe25680, {0x63c85b8, 0xc04f6fc780}, 0xc08e31a168, 0x1, 0xc03eb970a8, 0x519bbc, 0xc05396d140, 0xc048fcdc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04f6fc780}, 0x120, 0xc03eb97230, 0x5396d0e0, 0x0, 0x660069, 0xc05396d140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe25680, {0x63c85b8, 0xc04f6fc780}, 0xc05396d0e0, {{{{0x0, 0x0}, {0xc0778ff320, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f6fc780}, 0x6352410, 0x17839200) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fdbf900, {0x63c85b8, 0xc04f6fc780}, {0xc05396cfc0, {0x174301107ca83f7d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fdbf900, {0x63c85b8, 0xc04f6fc780}, 0xc082b2bf40, 0xc03eb979f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f6fc780}, 0x660089, {0xc05396cfc0, {0x174301107ca83f7d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fe65c00, {0x63c85b8, 0xc04f6fc780}, 0xc079621300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107ca83f7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107ca83f7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f6fc6f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f6fc6f0}, {0xc03eb993f8, 0x660089}, 0xc03eb99400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f6fc6f0}, {0xbebcce9248001642}, 0xc079621100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f6fc660}, 0xc079621100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}, 0xc087886620, 0xc043a91c38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}, 0x3, 0xc058616880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03eb99980}}}, {0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}, 0xc087886620, 0xc058616880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f6fc660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f6fc660}, {0x40, 0xc03eb99b70}, 0xc03eb99ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f6fc660}, {0x4e9d7a0, 0xc079621100}, 0xc087886620, 0xc043a91c38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f6fc660}, 0xc04d5f7740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05396cd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05396cd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805567 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc016c4d000, {0x63c85b8, 0xc066543d10}, 0xc048305600, 0xc06034b7c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc066543d10}, 0xc048305600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066543c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066543c50}, {0xc04e9033f8, 0x660089}, 0xc04e903400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066543c50}, {0x7fe42aae3ea0}, 0xc048305100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066543bc0}, 0xc048305100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}, 0xc063058760, 0xc067976f00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}, 0x3, 0xc033135400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04e903980}}}, {0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}, 0xc063058760, 0xc033135400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066543bc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066543bc0}, {0x40, 0xc04e903b70}, 0xc04e903ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}, 0x20300c, 0x20300c) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066543bc0}, {0x4e9d7a0, 0xc048305100}, 0xc063058760, 0xc067976f00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066543bc0}, 0xc035a1b1a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05c9e19e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05c9e19e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812442 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc031194500, {0x63c85b8, 0xc07d326270}, 0xc0afc18f90, 0x1, 0xc003f5b0a8, 0x519bbc, 0xc0737ae5c0, 0xc077fc9aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0477b4fc0, {0x63c85b8, 0xc07d326270}, 0xc0477b4ff0, 0xc003f5b1a8, 0x31, 0x40, 0xffffffffffffffff, 0xc0737ae5c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc031194500, {0x63c85b8, 0xc07d326270}, 0xc0737ae380, {{{{0xc05abf77a0, 0x1}, {0xc03fd82900, 0x1}}, {{0xc060236120, 0x5}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07d326270}, 0x63521a0, 0x655a2668) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc03107f9f0, {0x63c85b8, 0xc07d326270}, {0xc06b2e7560, {0x1743010ff5c022bb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc03107f9f0, {0x63c85b8, 0xc07d326270}, 0xc0634fe8c0, 0xc003f5b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07d326270}, 0x660089, {0xc06b2e7560, {0x1743010ff5c022bb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031189400, {0x63c85b8, 0xc07d326270}, 0xc0a5abf100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff5c022bb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff5c022bb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07d3261e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07d3261e0}, {0xc003f5d3f8, 0x660089}, 0xc003f5d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07d3261e0}, {0x7fe3c1556ac8}, 0xc0a5abef00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07d326120}, 0xc0a5abef00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}, 0xc02604ae80, 0xc0b26904c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}, 0x3, 0xc05fe4f9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003f5d980}}}, {0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}, 0xc02604ae80, 0xc05fe4f9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07d326120}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07d326120}, {0x40, 0xc003f5db70}, 0xc003f5dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07d326120}, {0x4e9d7a0, 0xc0a5abef00}, 0xc02604ae80, 0xc0b26904c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07d326120}, 0xc039559380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0137dad80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0137dad80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28700 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run(0xc0029e8a50, {0x63c85b8, 0xc0215c3aa0}, 0xc000d3ab40, {0x643def0, 0xc00a5f43c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:342 +0x1cb github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Receiver).PushUpdates(0xc01876a6c0, {0x643def0, 0xc00a5f43c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:89 +0x134 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb._SideTransport_PushUpdates_Handler({0x4cd47a0, 0xc01876a6c0}, {0x6400fe0, 0xc016dc0680}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:346 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}, 0xc018c97ae8, 0x51c46f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}, 0x3, 0xc007935f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0xc00102e340}}}, {0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}, 0xc018c97ae8, 0xc007935f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0215c3a10}, {0x460ef4, 0xc007b58bc8}, 0xc007b58b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}, 0xc018c97ae8, 0xc007935f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc00b755110}, 0xc018c97ae8, 0x51c46f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc019572680}, 0xc013e66ea0, 0xc007b7b7a0, 0x9181540, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019572680}, 0xc013e66ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812622 [semacquire]: sync.runtime_SemacquireMutex(0xc00bd42ea0, 0x3, 0xc00bd42ea0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc0433a4e40) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fe39f279800, 0x7f8d, 0x7fd4be1301c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc015e58520, 0x7fe39f279801, 0x7fe39f279820, 0x3612, 0x3617) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc015e63b80, {0x76fe9de, 0x3612}, 0x0, 0xc04fc8eff8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04fc8ed80, {0xffffffffffffff15, 0xc0433a50c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc04fc8ed80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc04fc8ed80, {0xc0d1536540, 0x18, 0x6d}, 0x13, 0x53) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04fc8ed80, {0xc0306a7b80, 0x18, 0x6d}, {0xc0d1536540, 0x0, 0xc04fc8ed80}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04fc8ed80, {0xc0306a7b80, 0x18, 0x74}, {0xc0d1536540, 0x18, 0x6d}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07a380d98, {0xc0306a7b80, 0x18, 0x74}, {0xc0d1536540, 0x18, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07a37f6c0, {0xc0d1536540, 0x0, 0xc0433a54e0}, 0x121ea30, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07a37f6c0, {0xc0306a7b80, 0x18, 0x6d}, {0xc0d1536540, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07a37f400, {0xc0d1536540, 0x18, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0830c8008, {{0xc08467ab28, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc082cfc300, {{0xc08467ab28, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0cd8eb000, {0x63c85b8, 0xc02f2c4780}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02f2c4780}, {0x64a6560, 0xc082cfc300}, {0xc08467ab28, 0x7, 0x18}, {0x17430110989b1cd8, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02f2c4780}, {0x6466ed8, 0xc0830c8000}, {0xc08467ab28, 0x17, 0x18}, {0x17430110989b1cd8, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc03c467f98}, {{0x17430110989b1cd8, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110989b1cd8, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0460dff00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b66881d8, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc02f2c4780}, 0xc0460dff00, 0xc03ac41400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc02f2c4780}, 0xc0460dff00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110989b1cd8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110989b1cd8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f2c4630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f2c4630}, {0xc0433ab3f8, 0x660089}, 0xc0433ab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f2c4630}, {0x0}, 0xc0460dfd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f2c4360}, 0xc0460dfd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}, 0xc03ebdda40, 0xc031cfb770) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}, 0x3, 0xc015b9ebc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0433ab980}}}, {0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}, 0xc03ebdda40, 0xc015b9ebc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f2c4360}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f2c4360}, {0x40, 0xc0433abb70}, 0xc0433abab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f2c4360}, {0x4e9d7a0, 0xc0460dfd00}, 0xc03ebdda40, 0xc031cfb770) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f2c4360}, 0xc01c342600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356ac60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356ac60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803786 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc06788b020}, 0xc015321aa0, 0x1, 0xc0771590a8, 0x519bbc, 0xc04b266ba0, 0xc04ba34f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06788b020}, 0x1e0, 0xc077159230, 0x4b266b40, 0x0, 0x660069, 0xc04b266ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc06788b020}, 0xc04b266b40, {{{{0x0, 0x0}, {0xc0905c9d40, 0x49}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06788b020}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc06788b020}, {0xc04d7a8b40, {0x174301103074bebc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc06788b020}, 0xc077741220, 0xc0771599f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06788b020}, 0x660089, {0xc04d7a8b40, {0x174301103074bebc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc06788b020}, 0xc0b67c0a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103074bebc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103074bebc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06788af90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06788af90}, {0xc07715b3f8, 0x660089}, 0xc07715b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06788af90}, {0xc0015221e0}, 0xc0b67c0800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06788af00}, 0xc0b67c0800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}, 0xc07cd49e40, 0xc077651b18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}, 0x3, 0xc07b340000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07715b980}}}, {0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}, 0xc07cd49e40, 0xc07b340000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06788af00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06788af00}, {0x40, 0xc07715bb70}, 0xc07715bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06788af00}, {0x4e9d7a0, 0xc0b67c0800}, 0xc07cd49e40, 0xc077651b18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06788af00}, 0xc0b45a4ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04d7a8a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04d7a8a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28620 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 209249240 [sync.Cond.Wait, 1 minutes]: sync.runtime_notifyListWait(0xc0290f3410, 0x991) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc03d71ea80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc03d71f0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc047b0d900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc047b0d900, {0x63c8510, 0xc0290f3440}, 0xc04aa5ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc031710e40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0290f3440}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc031710e40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802574 [runnable]: sync.runtime_SemacquireMutex(0xc07dada690, 0x8f, 0x7fe37d8be380) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc07dada538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc07dada560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc07ce68900, {0xffffffffffffff15, 0xc07dada6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc07ce68900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc07ce68900, {0xc0411d9560, 0x32, 0x53}, {0xc0796d0fc0, 0x32, 0x88}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0935bcd98, {0xc0411d9560, 0x32, 0x53}, {0xc0796d0fc0, 0x32, 0x88}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0935bb6c0, {0xc0796d0fc0, 0x88, 0x53}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0935bb6c0, {0xc0411d9560, 0x88, 0xc0796d0fc0}, {0xc0796d0fc0, 0xc06eb843c0, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0935bb400, {0xc0796d0fc0, 0x32, 0x88}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc064e52da8, {{0xc06eb843c0, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc053c90400, {{0xc070b17800, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0523c5800, {0x63c85b8, 0xc01f72a090}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01f72a090}, {0x64a6560, 0xc053c90400}, {0xc070b17800, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01f72a090}, {0x6466ed8, 0xc064e52b00}, {0xc070b17800, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0672e9848}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05113a500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f20800, {0x63c85b8, 0xc01f72a090}, 0xc05113a500, 0xc0774039a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f20800, {0x63c85b8, 0xc01f72a090}, 0xc05113a500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0780a5f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0780a5f80}, {0xc0d68328f0, 0x660089}, 0xc0d68328f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0780a5f80}, {0xc00102e4e0}, 0xc05113a400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0780a5f20}, 0xc05113a400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220b10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28149 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1({0x63c8510, 0xc007b0fa80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:116 +0x15a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219810816 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc03531b170}, 0xc02a0b2b28, 0x1, 0xc026f650a8, 0x519bbc, 0xc021842600, 0xc049093b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03531b170}, 0x120, 0xc026f65230, 0x218425a0, 0x0, 0x660069, 0xc021842600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc03531b170}, 0xc0218425a0, {{{{0x0, 0x0}, {0xc033c96240, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03531b170}, 0x6352410, 0x412adb48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc03531b170}, {0xc021842120, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc03531b170}, 0xc049edd9a0, 0xc026f659f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03531b170}, 0x660089, {0xc021842120, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc03531b170}, 0xc07d06f600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03531af60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03531af60}, {0xc026f673f8, 0x660089}, 0xc026f67400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03531af60}, {0x0}, 0xc07d06f400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03531ae40}, 0xc07d06f400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}, 0xc01544a940, 0xc0a7c03c08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}, 0x3, 0xc0c17042c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026f67980}}}, {0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}, 0xc01544a940, 0xc0c17042c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03531ae40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03531ae40}, {0x40, 0xc026f67b70}, 0xc026f67ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03531ae40}, {0x4e9d7a0, 0xc07d06f400}, 0xc01544a940, 0xc0a7c03c08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03531ae40}, 0xc01f734060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03bf5ca20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03bf5ca20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809908 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc012027400, {0x63c85b8, 0xc0209fdd40}, 0xc004c40200, 0xc07471ac80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc012027400, {0x63c85b8, 0xc0209fdd40}, 0xc004c40200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110681a1aa5, 0x0, 0x0}, 0xc031cfcec0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110681a1aa5, 0x0, 0x0}, 0xc031cfcec0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0209fdad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0209fdad0}, {0xc00df473f8, 0x660089}, 0xc00df47400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0209fdad0}, {0x7fe429311c20}, 0xc084513f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0209fccf0}, 0xc084513f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}, 0xc056c91980, 0xc01b3eb398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}, 0x3, 0xc04c170280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00df47980}}}, {0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}, 0xc056c91980, 0xc04c170280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0209fccf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0209fccf0}, {0x40, 0xc00df47b70}, 0xc00df47ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0209fccf0}, {0x4e9d7a0, 0xc084513f00}, 0xc056c91980, 0xc01b3eb398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0209fccf0}, 0xc0302a72c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0118c8ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0118c8ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805449 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc041ca1200}, 0xc015219998, 0x1, 0xc07b3f90a8, 0x519bbc, 0xc05f70b5c0, 0xc058cb7ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041ca1200}, 0x120, 0xc07b3f9230, 0x5f70b560, 0xc00bf42000, 0x18, 0xc05f70b5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc041ca1200}, 0xc05f70b560, {{{{0x0, 0x0}, {0xc033b72fc0, 0x1a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041ca1200}, 0x6352410, 0x24ff13f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc041ca1200}, {0xc05f70b440, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc041ca1200}, 0xc06191b400, 0xc07b3f99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041ca1200}, 0x660089, {0xc05f70b440, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc041ca1200}, 0xc055a4bc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041ca1170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ca1170}, {0xc07b3fb3f8, 0x660089}, 0xc07b3fb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041ca1170}, {0x7fe3945d5ca8}, 0xc055a4ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041ca10e0}, 0xc055a4ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}, 0xc01389a2e0, 0xc0465cbd28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}, 0x3, 0xc074875bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b3fb980}}}, {0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}, 0xc01389a2e0, 0xc074875bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041ca10e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ca10e0}, {0x40, 0xc07b3fbb70}, 0xc07b3fbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}, 0x20301d, 0x20301d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041ca10e0}, {0x4e9d7a0, 0xc055a4ba00}, 0xc01389a2e0, 0xc0465cbd28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041ca10e0}, 0xc02cab0960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f70b320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f70b320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28120 [select, 26 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1({0x63c85b8, 0xc02c8c7e90}) github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:632 +0x24b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802164 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc0a7b9f860}, 0xc056a7da00, 0xc075440d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0a7b9f860}, 0xc056a7da00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020a0b142, 0x0, 0x0}, 0xc07864e680, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020a0b142, 0x0, 0x0}, 0xc07864e680, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0a7b9f7d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a7b9f7d0}, {0xc0766d3a08, 0x660089}, 0xc0766d3a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0a7b9f7d0}, {0xc00102e4e0}, 0xc056a7d900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0a7b9f770}, 0xc056a7d900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc0a7b9f6e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0a7b9f6e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28955 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc014aa6fa0, {0xc02c48ce80, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc014aa6fa0, {0xc02c48ce80, 0xc0204b8c60, 0xc01117da80}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc02a7fccc0, {0xc02c48ce80, 0xc01117daf8, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc02a7fccc0}, {0xc02c48ce80, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc02c76e6c0, {0xc02c48ce80, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc02c48ce70, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc01117dc40, 0xc02c76e6c0, {0x0, 0x0}, 0x7fffffff, 0xc01117dda8, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0xc04783f2c0, {0x0, 0x0}, {0x4cfd920, 0xc04783f2c0}, 0x203011, 0xc01117dda8, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc014284b60, {0x4cfd920, 0xc04783f2c0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb.(*sideTransportPushUpdatesServer).Recv(0xc017cf0f10) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:365 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run.func1({0x63c85b8, 0xc02a7fd110}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:304 +0xc2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809174 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc03de0af60}, 0xc07e120cb0, {0x646cc48, 0xc03de0afc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc03de0af60}, 0xedb7a8350, 0xc066eaf320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc03de0af60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07bc0f0e0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc07e120ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc07bc0f0e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28329 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00bb50fa0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc014b02660) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 219807743 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc029693950}, 0xc09c378200, 0xc02f38e960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc029693950}, 0xc09c378200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105981709e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105981709e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029693890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029693890}, {0xc0058533f8, 0x660089}, 0xc005853400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029693890}, {0x4021ffde8883651f}, 0xc09c378000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029693800}, 0xc09c378000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}, 0xc01c519dc0, 0xc04309e6d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}, 0x3, 0xc0202af840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005853980}}}, {0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}, 0xc01c519dc0, 0xc0202af840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029693800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029693800}, {0x40, 0xc005853b70}, 0xc005853ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029693800}, {0x4e9d7a0, 0xc09c378000}, 0xc01c519dc0, 0xc04309e6d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029693800}, 0xc06dba5380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0765b25a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0765b25a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219143271 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc03380ea50, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0427e3800) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 28125 [select]: github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache.(*Cache).Start.func1({0xc01fae2600, 0xc02b1e3830}) github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache/txn_id_cache.go:144 +0xa9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812738 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812321 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809420 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0210be000, {0x63c85b8, 0xc0682a8a20}, 0xc058b30b00, 0xc04bf3b4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210be000, {0x63c85b8, 0xc0682a8a20}, 0xc058b30b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065f1cfe3, 0x0, 0x0}, 0xc062047810, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065f1cfe3, 0x0, 0x0}, 0xc062047810, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0682a8990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0682a8990}, {0xc0229d53f8, 0x660089}, 0xc0229d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0682a8990}, {0x0}, 0xc058b30900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0682a8900}, 0xc058b30900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}, 0xc0472a2c80, 0xc0337f87c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}, 0x3, 0xc01b65c580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0229d5980}}}, {0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}, 0xc0472a2c80, 0xc01b65c580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0682a8900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0682a8900}, {0x40, 0xc0229d5b70}, 0xc0229d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0682a8900}, {0x4e9d7a0, 0xc058b30900}, 0xc0472a2c80, 0xc0337f87c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0682a8900}, 0xc0b012f860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0849b5440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0849b5440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 15725 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc018727140, 0xc005939080, {0x643dbd8, 0xc01777c090}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc022d73e90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810357 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc01866e360}, 0xc03c2c38d8, 0x1, 0xc00b45d0a8, 0x519bbc, 0xc083c91980, 0xc050cffe00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0785d9f20, {0x63c85b8, 0xc01866e360}, 0xc093edcea0, 0xc00b45d1a8, 0x83c91980, 0x0, 0x2, 0xc083c91980, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc01866e360}, 0xc083c918c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc036670fc0, 0xa1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01866e360}, 0x6352260, 0x7b228a38) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc01866e360}, {0xc083c917a0, {0x174301103094392a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc01866e360}, 0xc02e9026e0, 0xc00b45d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01866e360}, 0x660089, {0xc083c917a0, {0x174301103094392a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc01866e360}, 0xc08a028e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01866e240}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01866e240}, {0xc00b45f3f8, 0x660089}, 0xc00b45f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01866e240}, {0x7fe13dd08ef0}, 0xc08a028b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01866e120}, 0xc08a028b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}, 0xc0412b4d40, 0xc02de9c4e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}, 0x3, 0xc038c784c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b45f980}}}, {0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}, 0xc0412b4d40, 0xc038c784c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01866e120}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01866e120}, {0x40, 0xc00b45fb70}, 0xc00b45fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01866e120}, {0x4e9d7a0, 0xc08a028b00}, 0xc0412b4d40, 0xc02de9c4e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01866e120}, 0xc0318e3140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc083c91680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc083c91680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807678 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0478052c0}, 0xc042ad3b30, 0x1, 0xc00b77b0a8, 0x519bbc, 0xc07a90ca80, 0xc0929f2c60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc02b7fab40, {0x63c85b8, 0xc0478052c0}, 0xc071a0d560, 0xc00b77b1a8, 0x7a90cae0, 0x0, 0x0, 0xc07a90ca80, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0478052c0}, 0xc07a90ca20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04d393200, 0x69}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc0478052c0}, 0x63523b0, 0x66762d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0478052c0}, {0x0, {0x174301105aac0ce0, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0478052c0}, 0xc04a99b360, 0xc00b77b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0478052c0}, 0x660089, {0x0, {0x174301105aac0ce0, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0478052c0}, 0xc0637c3900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aac0ce0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aac0ce0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301105aac0ce0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047805230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047805230}, {0xc00b77d3f8, 0x660089}, 0xc00b77d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047805230}, {0x7fe3b5821770}, 0xc0637c3700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0478051a0}, 0xc0637c3700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}, 0xc02ce7ece0, 0xc0293d8078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}, 0x3, 0xc053d7af80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b77d980}}}, {0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}, 0xc02ce7ece0, 0xc053d7af80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0478051a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0478051a0}, {0x40, 0xc00b77db70}, 0xc00b77dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0478051a0}, {0x4e9d7a0, 0xc0637c3700}, 0xc02ce7ece0, 0xc0293d8078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0478051a0}, 0xc0327d0a20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0790799e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0790799e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812540 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc04bf64f60}, 0xc0321fc270, 0x1, 0xc03f7650a8, 0x519bbc, 0xc0346a3bc0, 0xc035f0cf60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04c2009c0, {0x63c85b8, 0xc04bf64f60}, 0xc05529a240, 0xc03f7651a8, 0x346a3bc0, 0x0, 0x0, 0xc0346a3bc0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc04bf64f60}, 0xc0346a3b00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc00ef146c0, 0x51}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04bf64f60}, 0x6352260, 0x204d3218) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc04bf64f60}, {0xc0346a39e0, {0x174301104be59ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc04bf64f60}, 0xc072301180, 0xc03f7659f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04bf64f60}, 0x660089, {0xc0346a39e0, {0x174301104be59ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc04bf64f60}, 0xc04aa85200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104be59ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104be59ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bf64ed0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf64ed0}, {0xc03f7673f8, 0x660089}, 0xc03f767400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bf64ed0}, {0x7ba2202f9c1ee}, 0xc04aa85000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bf64e40}, 0xc04aa85000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}, 0xc0ccf7f720, 0xc038161638) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}, 0x3, 0xc02a52ec80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03f767980}}}, {0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}, 0xc0ccf7f720, 0xc02a52ec80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bf64e40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf64e40}, {0x40, 0xc03f767b70}, 0xc03f767ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bf64e40}, {0x4e9d7a0, 0xc04aa85000}, 0xc0ccf7f720, 0xc038161638) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bf64e40}, 0xc05c2fa9c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdd9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdd9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795623 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc03e394cc0}, 0xc0209a2cb0, {0x646cc48, 0xc03e394d20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc03e394cc0}, 0xedb7a834e, 0xc05ed4fd40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc03e394cc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc08209b680}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0209a2ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc08209b680}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809098 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc06ee63470}, 0xc061b3a840, 0x1, 0x0, 0x1, 0xc087325a40, 0xc07389eba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06ee63470}, 0x120, 0xc00fc9a728, 0x873259e0, 0x0, 0x660069, 0xc087325a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc06ee63470}, 0xc0873259e0, {{{{0x0, 0x0}, {0xc036692480, 0x29}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06ee63470}, 0x6352410, 0x42a0eaf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc06ee63470}, {0xc0873258c0, {0x1743011061978db5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc06ee63470}, 0xc070dc15e0, 0xc0766b2ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06ee63470}, 0x660089, {0xc0873258c0, {0x1743011061978db5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc06ee63470}, 0xc005251400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061978db5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061978db5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ee633e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ee633e0}, {0xc0936388f0, 0x660089}, 0xc0936388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ee633e0}, {0x24}, 0xc038ac5400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06ee63380}, 0xc038ac5400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06ee63140}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219153677 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc071e5ac30, {0xc0354c71f8, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc071e5ac30, {0xc0354c71f8, 0xc00fb0a2a0, 0xc0006eca80}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc026394f90, {0xc0354c71f8, 0xc0006ecaf8, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc026394f90}, {0xc0354c71f8, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0770ede60, {0xc0354c71f8, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc0354c71e8, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x58, 0xc0770ede60, {0x0, 0x0}, 0x7fffffff, 0xc0006ecda8, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0xc04d73ad80, {0x0, 0x0}, {0x4cfd920, 0xc04d73ad80}, 0x203013, 0xc0006ecda8, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc026af25b0, {0x4cfd920, 0xc04d73ad80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb.(*sideTransportPushUpdatesServer).Recv(0xc079b7d270) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:365 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run.func1({0x63c85b8, 0xc026395200}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:304 +0xc2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802158 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fefcdc0, {0x63c85b8, 0xc04d17e330}, 0xc03a855b18, 0x1, 0xc01b7b50a8, 0x519bbc, 0xc020e38a80, 0xc05b782960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04d17e330}, 0x120, 0xc01b7b5230, 0x20e38a20, 0x0, 0x660069, 0xc020e38a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fefcdc0, {0x63c85b8, 0xc04d17e330}, 0xc020e38a20, {{{{0x0, 0x0}, {0xc07a1cd7a0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d17e330}, 0x6352410, 0x351a9978) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea3400, {0x63c85b8, 0xc04d17e330}, {0xc020e387e0, {0x174301101b171816, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea3400, {0x63c85b8, 0xc04d17e330}, 0xc08e181540, 0xc01b7b59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d17e330}, 0x660089, {0xc020e387e0, {0x174301101b171816, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff32000, {0x63c85b8, 0xc04d17e330}, 0xc06d00f700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b171816, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b171816, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d17e2a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d17e2a0}, {0xc01b7b73f8, 0x660089}, 0xc01b7b7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d17e2a0}, {0x36}, 0xc06d00f500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d17e210}, 0xc06d00f500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}, 0xc04ecdb140, 0xc0302f5f50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}, 0x3, 0xc06e1c7540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01b7b7980}}}, {0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}, 0xc04ecdb140, 0xc06e1c7540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d17e210}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d17e210}, {0x40, 0xc01b7b7b70}, 0xc01b7b7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d17e210}, {0x4e9d7a0, 0xc06d00f500}, 0xc04ecdb140, 0xc0302f5f50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d17e210}, 0xc04ad09860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc020e385a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc020e385a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807405 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc0379b05a0}, 0xc092037900, 0xc05efdf0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0379b05a0}, 0xc092037900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110572a0114, 0x0, 0x0}, 0xc0907f4490, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110572a0114, 0x0, 0x0}, 0xc0907f4490, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0379b0510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0379b0510}, {0xc003ca93f8, 0x660089}, 0xc003ca9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0379b0510}, {0x7fe3b591b1a0}, 0xc092037700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0379b0480}, 0xc092037700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}, 0xc007b4eac0, 0xc046971980) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}, 0x3, 0xc039e7b400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003ca9980}}}, {0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}, 0xc007b4eac0, 0xc039e7b400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0379b0480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0379b0480}, {0x40, 0xc003ca9b70}, 0xc003ca9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0379b0480}, {0x4e9d7a0, 0xc092037700}, 0xc007b4eac0, 0xc046971980) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0379b0480}, 0xc05205fec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc085d54240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc085d54240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808485 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6140, {0x63c85b8, 0xc01b2fb740}, 0xc081cd64e0, 0x1, 0x0, 0x1, 0xc05c3b3c80, 0xc05c3b2840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01b2fb740}, 0x120, 0xc007568728, 0x5c3b3c20, 0x0, 0x660069, 0xc05c3b3c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6140, {0x63c85b8, 0xc01b2fb740}, 0xc05c3b3c20, {{{{0x0, 0x0}, {0xc090f2b8c0, 0xa}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01b2fb740}, 0x6352410, 0x42a0e260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddbd0, {0x63c85b8, 0xc01b2fb740}, {0xc05adad680, {0x174301105c9acd56, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddbd0, {0x63c85b8, 0xc01b2fb740}, 0xc0ab4d7360, 0xc089d54ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01b2fb740}, 0x660089, {0xc05adad680, {0x174301105c9acd56, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099c00, {0x63c85b8, 0xc01b2fb740}, 0xc086ec8f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c9acd56, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c9acd56, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b2fb6b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b2fb6b0}, {0xc0d68328f0, 0x660089}, 0xc0d68328f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b2fb6b0}, {0xc00102e4e0}, 0xc086ec8d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b2fb650}, 0xc086ec8d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0208a5b60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804552 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc023b91680, {0x63c85b8, 0xc03a015500}, 0xc06292f5f0, 0x1, 0xc076ebd0a8, 0x519bbc, 0xc06440c4e0, 0xc020d614a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03a015500}, 0x120, 0xc076ebd230, 0x6440c480, 0x0, 0x660069, 0xc06440c4e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc023b91680, {0x63c85b8, 0xc03a015500}, 0xc06440c480, {{{{0x0, 0x0}, {0xc0517b3560, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03a015500}, 0x6352410, 0x34331258) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc023a89590, {0x63c85b8, 0xc03a015500}, {0xc06440c360, {0x1743011015406c30, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc023a89590, {0x63c85b8, 0xc03a015500}, 0xc0688e8dc0, 0xc076ebd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03a015500}, 0x660089, {0xc06440c360, {0x1743011015406c30, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023bf0800, {0x63c85b8, 0xc03a015500}, 0xc0a4f85b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011015406c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011015406c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03a015470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a015470}, {0xc076ebf3f8, 0x660089}, 0xc076ebf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03a015470}, {0x7fe3bdd4cb98}, 0xc0a4f85900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a015380}, 0xc0a4f85900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}, 0xc039a58220, 0xc028551950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}, 0x3, 0xc044f49ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076ebf980}}}, {0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}, 0xc039a58220, 0xc044f49ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03a015380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a015380}, {0x40, 0xc076ebfb70}, 0xc076ebfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03a015380}, {0x4e9d7a0, 0xc0a4f85900}, 0xc039a58220, 0xc028551950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03a015380}, 0xc06be25bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06440c240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06440c240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 27337 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc01876a900, {0x63c8510, 0xc0071fdd00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:319 +0x997 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x63c85b8, 0xc02caab3b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:189 +0x274 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219213809 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3dff50, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc060f0b780, 0xc0befe5000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc060f0b780, {0xc0befe5000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc060f0b780, {0xc0befe5000, 0x735, 0xc07516b180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03d603778, {0xc0befe5000, 0x7eda79, 0xc0befe5005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01c45ea20, {0xc0befe5000, 0xc07b596c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc01b5eef78, {0xc0befe5000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc05f19e5f8, {0x631b900, 0xc01b5eef78}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc05f19e380, {0x7fe42ab59310, 0xc01c45ea20}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc05f19e380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc05f19e380, {0xc08b443000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc085be0990, {0xc08b443000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05ff3c088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05ff3c088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05ff3c660, {0x634aac0, 0xc05ff3c088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05ff3c000, 0xc07946d310, 0xc082ae742b, 0xc082ae742d, {0x635bec0, 0xc0610e10e0}, 0xc000d6ae00, 0xc07864e478, 0xc082ae7429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05ff3c000, {0x63c85b8, 0xc032697d40}, 0xc07946d300, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc01acb3026, 0x8}}, 0x0, 0xc032697ce0, 0xc032697d10, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc032697d40}, {0x643d810, 0xc01c45ea20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07ae38c00}, {0x643d810, 0xc01c45ea20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07ae38c00}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800550 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc094ac2548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc094ac25b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc087250cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc050841c00, {0xc048309680, 0x2b, 0xa0}, {0xc06b5a2230, 0x2b, 0x6d}, 0x1, 0xc) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc087250b88, {0xc048309680, 0x1, 0x6d}, {0xc06b5a2230, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc087250b88, {0xc048309680, 0x2b, 0xa0}, {0xc06b5a2230, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0872502c0, {0xc06b5a2230, 0x6d, 0xa0}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0872502c0, {0xc048309680, 0x2b, 0xc06b5a2230}, {0xc06b5a2230, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc087250000, {0xc06b5a2230, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0c33bf708, {{0xc0717fa5d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc00c23aa00, {{0xc0717fa5d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc059db2400, {0x63c85b8, 0xc05a160c60}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc05a160c60}, {0x64a6560, 0xc00c23aa00}, {0xc0717fa5d0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc05a160c60}, {0x6466ed8, 0xc0c33bf700}, {0xc0717fa5d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc074391a28}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05be30800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020af1000, {0x63c85b8, 0xc05a160c60}, 0xc05be30800, 0xc0777e3900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020af1000, {0x63c85b8, 0xc05a160c60}, 0xc05be30800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05a160bd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05a160bd0}, {0xc07d2568f0, 0x660089}, 0xc07d2568f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05a160bd0}, {0xc00102e4e0}, 0xc05be30700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05a160a80}, 0xc05be30700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7bb30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28417 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 219806647 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc08d3c1c80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28031 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run(0xc0198f4370, {0x63c85b8, 0xc02c5b26f0}, 0xc000d3ab40, {0x643def0, 0xc01c38e570}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:342 +0x1cb github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Receiver).PushUpdates(0xc01876a6c0, {0x643def0, 0xc01c38e570}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:89 +0x134 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb._SideTransport_PushUpdates_Handler({0x4cd47a0, 0xc01876a6c0}, {0x6400fe0, 0xc006db99a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:346 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}, 0xc022c5e930, 0x51c46f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}, 0x3, 0xc007b97ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x16f0987}}}, {0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}, 0xc022c5e930, 0xc007b97ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c5b2660}, {0x460ef4, 0xc01fac8bc8}, 0xc01fac8b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}, 0xc022c5e930, 0xc007b97ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc006dfcd00}, 0xc022c5e930, 0x51c46f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc0198ce9c0}, 0xc017ee1560, 0xc007b7b7a0, 0x9181540, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0198ce9c0}, 0xc017ee1560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800868 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02c4a8c00, {0x63c85b8, 0xc09a5ba120}, 0xc0401ea100, 0xc03e507e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc09a5ba120}, 0xc0401ea100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110165b67fe, 0x0, 0x0}, 0xc063dcef60, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110165b67fe, 0x0, 0x0}, 0xc063dcef60, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09a5ba090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09a5ba090}, {0xc03af113f8, 0x660089}, 0xc03af11400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09a5ba090}, {0x7fe3ba7a9488}, 0xc04dfe1a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09a5ba000}, 0xc04dfe1a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}, 0xc02b51b280, 0xc0432a4648) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}, 0x3, 0xc0266fd4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03af11980}}}, {0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}, 0xc02b51b280, 0xc0266fd4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09a5ba000}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09a5ba000}, {0x40, 0xc03af11b70}, 0xc03af11ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09a5ba000}, {0x4e9d7a0, 0xc04dfe1a00}, 0xc02b51b280, 0xc0432a4648) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09a5ba000}, 0xc035ec9aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05c2538c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05c2538c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805396 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc0786341e0}, 0xc0525c0090, 0x1, 0xc07abb10a8, 0x519bbc, 0xc05ad4f4a0, 0xc05ad4ef00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0786341e0}, 0x120, 0xc07abb1230, 0x5ad4f440, 0xc00c387380, 0x18, 0xc05ad4f4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc0786341e0}, 0xc05ad4f440, {{{{0x0, 0x0}, {0xc0b4c8ea20, 0x1b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0786341e0}, 0x6352410, 0x1f8e1ff8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc0786341e0}, {0xc05ad4f320, {0x1743011041c6d918, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc0786341e0}, 0xc07beec640, 0xc07abb19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0786341e0}, 0x660089, {0xc05ad4f320, {0x1743011041c6d918, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0786341e0}, 0xc08b702c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041c6d918, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041c6d918, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078634150}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078634150}, {0xc07abb33f8, 0x660089}, 0xc07abb3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078634150}, {0x7fde181c6bd0}, 0xc08b702a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0786340c0}, 0xc08b702a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}, 0xc0a0c54080, 0xc0525c0fc0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}, 0x3, 0xc0455d1480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07abb3980}}}, {0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}, 0xc0a0c54080, 0xc0455d1480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0786340c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0786340c0}, {0x40, 0xc07abb3b70}, 0xc07abb3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}, 0x203011, 0x203011) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0786340c0}, {0x4e9d7a0, 0xc08b702a00}, 0xc0a0c54080, 0xc0525c0fc0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0786340c0}, 0xc02fd385a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ad4f200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ad4f200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812791 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807622 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc06c998690}, 0xc05fe66a50, 0x1, 0xc00672f0a8, 0x519bbc, 0xc062beda40, 0xc07952b380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06c998690}, 0x120, 0xc00672f230, 0x62bed9e0, 0x0, 0x660069, 0xc062beda40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc06c998690}, 0xc062bed9e0, {{{{0x0, 0x0}, {0xc03fb8b440, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06c998690}, 0x6352410, 0x6c099bd8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc06c998690}, {0xc062bed8c0, {0x1743011058606be6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc06c998690}, 0xc04d818640, 0xc00672f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06c998690}, 0x660089, {0xc062bed8c0, {0x1743011058606be6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc06c998690}, 0xc098cabd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058606be6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058606be6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06c998600}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c998600}, {0xc0067313f8, 0x660089}, 0xc006731400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06c998600}, {0x3feffe1328e8ab1a}, 0xc098cabb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06c998570}, 0xc098cabb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}, 0xc031c7fa20, 0xc06cd8bcf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}, 0x3, 0xc06754c300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006731980}}}, {0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}, 0xc031c7fa20, 0xc06754c300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06c998570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c998570}, {0x40, 0xc006731b70}, 0xc006731ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06c998570}, {0x4e9d7a0, 0xc098cabb00}, 0xc031c7fa20, 0xc06cd8bcf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06c998570}, 0xc04601a180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0579f4900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0579f4900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28616 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0003df278, 0xfc619) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xadd280) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc0003df260, {0x63c8510, 0xc030473000}, 0x327c1) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xde github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x63c85b8, 0xc02cc73560}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x357 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808812 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0357c0, {0x63c85b8, 0xc03ce77b60}, 0xc0420add40, 0x1, 0xc00590d0a8, 0x519bbc, 0xc06c8d5920, 0xc065776600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03ce77b60}, 0x120, 0xc00590d230, 0x6c8d58c0, 0x0, 0x660069, 0xc06c8d5920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0357c0, {0x63c85b8, 0xc03ce77b60}, 0xc06c8d58c0, {{{{0x0, 0x0}, {0xc0cee1d9e0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03ce77b60}, 0x6352410, 0x603b8578) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd770, {0x63c85b8, 0xc03ce77b60}, {0xc06c8d5680, {0x174301105fd3e432, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd770, {0x63c85b8, 0xc03ce77b60}, 0xc05ad9e780, 0xc00590d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03ce77b60}, 0x660089, {0xc06c8d5680, {0x174301105fd3e432, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc03ce77b60}, 0xc049b20400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105fd3e432, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105fd3e432, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03ce77aa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ce77aa0}, {0xc00590f3f8, 0x660089}, 0xc00590f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03ce77aa0}, {0x736d726f6674616c}, 0xc049b20200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03ce77a10}, 0xc049b20200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}, 0xc073ff40c0, 0xc0506a7ae8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}, 0x3, 0xc033bdb6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00590f980}}}, {0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}, 0xc073ff40c0, 0xc033bdb6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03ce77a10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ce77a10}, {0x40, 0xc00590fb70}, 0xc00590fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03ce77a10}, {0x4e9d7a0, 0xc049b20200}, 0xc073ff40c0, 0xc0506a7ae8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03ce77a10}, 0xc07d817380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06c8d5560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06c8d5560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28586 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219789419 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc01f9cae40}, 0xc0057facb0, {0x646cc48, 0xc01f9caea0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc01f9cae40}, 0xedb7a834d, 0xc03f7d4120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc01f9cae40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03c105b60}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0057faef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc03c105b60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 16181 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001306580, {0x63c8510, 0xc0036c5c00}, 0xc0194c3fb0, 0xc0161b3e60) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:331 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x63c8510, 0xc0036c5c00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:288 +0x38 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 28690 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219811141 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc059284d50}, 0xc042e8cb10, 0x1, 0xc0048a50a8, 0x519bbc, 0xc06b4a6de0, 0xc05ed4e180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc059284d50}, 0x120, 0xc0048a5230, 0x6b4a6d80, 0x0, 0x660069, 0xc06b4a6de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc059284d50}, 0xc06b4a6d80, {{{{0x0, 0x0}, {0xc01acbd200, 0x64}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc059284d50}, 0x6352410, 0x3a7e0de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc059284d50}, {0xc06b4a6c60, {0x17430110725dba50, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc059284d50}, 0xc05ad73f40, 0xc0048a59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc059284d50}, 0x660089, {0xc06b4a6c60, {0x17430110725dba50, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc059284d50}, 0xc06c9cea00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110725dba50, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110725dba50, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059284cc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059284cc0}, {0xc0048a73f8, 0x660089}, 0xc0048a7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059284cc0}, {0xc0015221e0}, 0xc06c9ce800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc059284c30}, 0xc06c9ce800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}, 0xc058229860, 0xc0541fe7b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}, 0x3, 0xc0399792c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0048a7980}}}, {0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}, 0xc058229860, 0xc0399792c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc059284c30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059284c30}, {0x40, 0xc0048a7b70}, 0xc0048a7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc059284c30}, {0x4e9d7a0, 0xc06c9ce800}, 0xc058229860, 0xc0541fe7b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc059284c30}, 0xc0380c6840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00da5f7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00da5f7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801302 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b26c000, {0x63c85b8, 0xc03df9d320}, 0xc00c824900, 0xc071dac3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc03df9d320}, 0xc00c824900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ae289ef, 0x0, 0x0}, 0xc06cefcda0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ae289ef, 0x0, 0x0}, 0xc06cefcda0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03df9d290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03df9d290}, {0xc0526b93f8, 0x660089}, 0xc0526b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03df9d290}, {0x7fe423ec24a0}, 0xc060f15f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03df9d200}, 0xc060f15f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}, 0xc05914e340, 0xc01fc64900) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}, 0x3, 0xc03ce99240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0526b9980}}}, {0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}, 0xc05914e340, 0xc03ce99240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03df9d200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03df9d200}, {0x40, 0xc0526b9b70}, 0xc0526b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03df9d200}, {0x4e9d7a0, 0xc060f15f00}, 0xc05914e340, 0xc01fc64900) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03df9d200}, 0xc01bce5440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc077120d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc077120d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812562 [semacquire]: sync.runtime_SemacquireMutex(0xc0438c4df8, 0x66, 0xc00074ad10) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0x3c8be16, 0x3117cc0, 0x7fdfbfbb5900) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x1199135, 0xc0438c4e68, 0xc0438c4ee8, 0x11866c6) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x4d866e, 0x9185600}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).init(0xc048319680, 0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1229 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2157 +0xdb github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a528c0, 0xc0438c5188, 0xc0892d2de8, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc0892d2f00, 0x0, 0xc0892d2f00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0892d2d98, 0xc0734106e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0892d2d98, {0xc0368125a0, 0x1f, 0x9a}, {0xc085bca000, 0x1f, 0x8c}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0892d16c0, {0xc085bca000, 0x2, 0x3}, 0x1c, 0xc8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0892d16c0, {0xc0368125a0, 0x8c, 0xc09d434fc0}, {0xc085bca000, 0xc09d434fc0, 0x1e}, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0892d1400, {0xc085bca000, 0x1f, 0x8c}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc057174928, {{0xc09d434fc0, 0x1e, 0xc0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0b73d6d00, {{0xc0578ee690, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04b392000, {0x63c85b8, 0xc056615c20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056615c20}, {0x64a6560, 0xc0b73d6d00}, {0xc0578ee690, 0x0, 0x18}, {0x1743011095958a2b, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056615c20}, {0x6466ed8, 0xc057174680}, {0xc0578ee690, 0x17, 0x18}, {0x1743011095958a2b, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0672e9f50}, {{0x1743011095958a2b, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011095958a2b, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046449500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b362ef2b, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc056615c20}, 0xc046449500, 0xc0413c7ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc056615c20}, 0xc046449500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011095958a2b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011095958a2b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056615ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056615ad0}, {0xc0438cb3f8, 0x660089}, 0xc0438cb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056615ad0}, {0x3feffe1328e8ab1a}, 0xc046449300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056615a40}, 0xc046449300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}, 0xc076de6040, 0xc044abe1b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}, 0x3, 0xc071e55980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0438cb980}}}, {0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}, 0xc076de6040, 0xc071e55980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056615a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056615a40}, {0x40, 0xc0438cbb70}, 0xc0438cbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056615a40}, {0x4e9d7a0, 0xc046449300}, 0xc076de6040, 0xc044abe1b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056615a40}, 0xc058eb0000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea3560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea3560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812638 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3dfaa0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06d853c00, 0xc011600800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06d853c00, {0xc011600800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06d853c00, {0xc011600800, 0xc05ed95e90, 0x0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc033a92d90, {0xc011600800, 0x4a2372, 0x4d889e}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04d73b020, {0xc011600800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc015f08258, {0xc011600800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0787b25f8, {0x631b900, 0xc015f08258}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0787b2380, {0x7fe42ab59310, 0xc04d73b020}, 0xc02e104ba0) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0787b2380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0787b2380, {0xc057704000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc05bec8d50, {0xc057704000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc063610b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc063610b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0636110e0, {0x634aac0, 0xc063610b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc063610a80, 0xc0a0eb9d10, 0xc06cf7742b, 0xc06cf7742d, {0x635bec0, 0xc049a25710}, 0xc000d6ae00, 0xc012352d18, 0xc06cf77429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc063610a80, {0x63c85b8, 0xc06c832150}, 0xc0a0eb9d00, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc051c91037, 0x8}}, 0x0, 0xc06c8320f0, 0xc06c832120, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06c832150}, {0x643d810, 0xc04d73b020}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc073d147b0}, {0x643d810, 0xc04d73b020}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc073d147b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801968 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0183a4c00, {0x63c85b8, 0xc0ae218ae0}, 0xc06cd04a00, 0xc095d5d7c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0183a4c00, {0x63c85b8, 0xc0ae218ae0}, 0xc06cd04a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff4948f0e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff4948f0e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0ae218a50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ae218a50}, {0xc0152b33f8, 0x660089}, 0xc0152b3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0ae218a50}, {0xfec73c6679d80bea}, 0xc06cd04700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0ae218960}, 0xc06cd04700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}, 0xc0621ffba0, 0xc064234888) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}, 0x3, 0xc07d3202c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0152b3980}}}, {0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}, 0xc0621ffba0, 0xc07d3202c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0ae218960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ae218960}, {0x40, 0xc0152b3b70}, 0xc0152b3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0ae218960}, {0x4e9d7a0, 0xc06cd04700}, 0xc0621ffba0, 0xc064234888) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0ae218960}, 0xc027453320, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04cb0f9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04cb0f9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800358 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe797c8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0571ba700, 0xc0331f4000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0571ba700, {0xc0331f4000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0571ba700, {0xc0331f4000, 0x735, 0xc067d34140}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc065becb20, {0xc0331f4000, 0x7eda79, 0xc0331f4005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04dc45c20, {0xc0331f4000, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03c545ad0, {0xc0331f4000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04be29af8, {0x631b900, 0xc03c545ad0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04be29880, {0x7fe42ab59310, 0xc04dc45c20}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04be29880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04be29880, {0xc02a7e1000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03a616528, {0xc02a7e1000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc099d98b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc099d98b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc099d990e0, {0x634aac0, 0xc099d98b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc099d98a80, 0xc0836355a0, 0xc00496342b, 0xc00496342d, {0x635bec0, 0xc0109d0240}, 0xc000d6ae00, 0xc06506fc88, 0xc004963429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc099d98a80, {0x63c85b8, 0xc01cfd8360}, 0xc083635590, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06f1d403e, 0x8}}, 0x0, 0xc01cfd82a0, 0xc01cfd8300, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc01cfd8360}, {0x643d810, 0xc04dc45c20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc046d66240}, {0x643d810, 0xc04dc45c20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc046d66240}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807631 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e068000, {0x63c85b8, 0xc07c060960}, 0xc06212ec00, 0xc06bf1d4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc07c060960}, 0xc06212ec00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d2ea197, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d2ea197, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c0608d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c0608d0}, {0xc007f893f8, 0x660089}, 0xc007f89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c0608d0}, {0x17a1c0a1fdcae4b6}, 0xc06212ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c060840}, 0xc06212ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}, 0xc0d55ba2c0, 0xc03bb18a98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}, 0x3, 0xc0141e5480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007f89980}}}, {0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}, 0xc0d55ba2c0, 0xc0141e5480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c060840}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c060840}, {0x40, 0xc007f89b70}, 0xc007f89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c060840}, {0x4e9d7a0, 0xc06212ea00}, 0xc0d55ba2c0, 0xc03bb18a98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c060840}, 0xc04e119980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc053ff4ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc053ff4ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 15952 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc002261260, {0x63c8510, 0xc0029365c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:310 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc002261260, {0x63c85b8, 0xc0232f0480}, 0xc023180a20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:335 +0xbb github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x63c85b8, 0xc0232f0480}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28271 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1({0x63c8510, 0xc02aec1080}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:116 +0x15a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219812701 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805271 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016f5da40, {0x63c85b8, 0xc01f16b110}, 0xc0644d9008, 0x1, 0x708017ddfea25d, 0x37e11d600, 0xc058e75380, 0xc059e74060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01f16b110}, 0x120, 0xc071c87230, 0x58e75320, 0xc00c8ba1a0, 0x18, 0xc058e75380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016f5da40, {0x63c85b8, 0xc01f16b110}, 0xc058e75320, {{{{0x0, 0x0}, {0xc05eb8f320, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01f16b110}, 0x6352410, 0x3f6a1a28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016fda500, {0x63c85b8, 0xc01f16b110}, {0xc058e75200, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016fda500, {0x63c85b8, 0xc01f16b110}, 0xc05fb7ee60, 0xc071c879f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01f16b110}, 0x660089, {0xc058e75200, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017050000, {0x63c85b8, 0xc01f16b110}, 0xc06702ee00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01f16b050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01f16b050}, {0xc071c893f8, 0x660089}, 0xc071c89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01f16b050}, {0xc0015221e0}, 0xc06702ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01f16af90}, 0xc06702ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}, 0xc065da6780, 0xc0644d8fd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}, 0x3, 0xc074900dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc071c89980}}}, {0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}, 0xc065da6780, 0xc074900dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01f16af90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01f16af90}, {0x40, 0xc071c89b70}, 0xc071c89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}, 0x20301d, 0x20301d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01f16af90}, {0x4e9d7a0, 0xc06702ec00}, 0xc065da6780, 0xc0644d8fd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01f16af90}, 0xc0c4129920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc058e750e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc058e750e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806186 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc03aa37650}, 0xc0530ef700, 0xc0541f25a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc03aa37650}, 0xc0530ef700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c836127, 0x0, 0x0}, 0xc078054880, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c836127, 0x0, 0x0}, 0xc078054880, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03aa37500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03aa37500}, {0xc0075733f8, 0x660089}, 0xc007573400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03aa37500}, {0x2ea8494167a38a3b}, 0xc0530ef500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03aa37470}, 0xc0530ef500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}, 0xc0834a39c0, 0xc08d926090) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}, 0x3, 0xc040b06bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007573980}}}, {0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}, 0xc0834a39c0, 0xc040b06bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03aa37470}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03aa37470}, {0x40, 0xc007573b70}, 0xc007573ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03aa37470}, {0x4e9d7a0, 0xc0530ef500}, 0xc0834a39c0, 0xc08d926090) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03aa37470}, 0xc08ce46e40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03a66f0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03a66f0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802045 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f8b4280, {0x63c85b8, 0xc0606fc6f0}, 0xc0387e5068, 0x1, 0xc0601d10a8, 0x519bbc, 0xc04b25f260, 0xc03f04f6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0606fc6f0}, 0x120, 0xc0601d1230, 0x4b25f200, 0x0, 0x660069, 0xc04b25f260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f8b4280, {0x63c85b8, 0xc0606fc6f0}, 0xc04b25f200, {{{{0x0, 0x0}, {0xc0118c9560, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0606fc6f0}, 0x6352410, 0x76af7208) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f6dd220, {0x63c85b8, 0xc0606fc6f0}, {0xc04b25f0e0, {0x174301101cc965ab, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f6dd220, {0x63c85b8, 0xc0606fc6f0}, 0xc0736d0aa0, 0xc0601d19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0606fc6f0}, 0x660089, {0xc04b25f0e0, {0x174301101cc965ab, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f8ac400, {0x63c85b8, 0xc0606fc6f0}, 0xc02754c000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101cc965ab, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101cc965ab, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0606fc660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0606fc660}, {0xc0601d33f8, 0x660089}, 0xc0601d3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0606fc660}, {0x4361546864786153}, 0xc052b23e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0606fc5d0}, 0xc052b23e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}, 0xc03c73ae20, 0xc06af89e78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}, 0x3, 0xc0cec297c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0601d3980}}}, {0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}, 0xc03c73ae20, 0xc0cec297c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0606fc5d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0606fc5d0}, {0x40, 0xc0601d3b70}, 0xc0601d3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0606fc5d0}, {0x4e9d7a0, 0xc052b23e00}, 0xc03c73ae20, 0xc06af89e78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0606fc5d0}, 0xc037facd20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04b25ed80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04b25ed80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809955 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).ResolveIndeterminateCommit(0xc00d502dc0, {0x63c85b8, 0xc023208ba0}, 0xc0171ce480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:100 +0x2f7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleIndeterminateCommitError(0xc021c42780, {0x63c85b8, 0xc023208ba0}, 0xc05feb6b00, 0xc04c408180, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:743 +0x4f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c6c800, {0x63c85b8, 0xc023208ba0}, 0xc05feb6b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:503 +0x8d3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110103f8259, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110103f8259, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023208b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023208b10}, {0xc00a7633f8, 0x660089}, 0xc00a763400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023208b10}, {0x537a6931626a7037}, 0xc05feb6900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0232089c0}, 0xc05feb6900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}, 0xc032ba46e0, 0xc0321fccd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}, 0x3, 0xc071c633c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a763980}}}, {0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}, 0xc032ba46e0, 0xc071c633c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0232089c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0232089c0}, {0x40, 0xc00a763b70}, 0xc00a763ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0232089c0}, {0x4e9d7a0, 0xc05feb6900}, 0xc032ba46e0, 0xc0321fccd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0232089c0}, 0xc018de7020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc00e2c57a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc00e2c57a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804397 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc031d677d0}, 0xc0164c3b18, 0x1, 0x1, 0x1, 0xc0516962a0, 0xc029f6bf20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc058c8aab0, {0x63c85b8, 0xc031d677d0}, 0xc04fbe8b40, 0xc0544946a0, 0x516962a0, 0x0, 0x0, 0xc0516962a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc031d677d0}, 0xc0516961e0, {{{{0xc0ad639440, 0x2e}, {0xc0905c9d40, 0x49}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x22, {0x63c85b8, 0xc031d677d0}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc031d677d0}, {0xc060e62240, {0x17430110353d085b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc031d677d0}, 0xc06891a140, 0xc07b0d4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc031d677d0}, 0x660089, {0xc060e62240, {0x17430110353d085b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc031d677d0}, 0xc06b8bdc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110353d085b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110353d085b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031d67740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031d67740}, {0xc0036f68f0, 0x660089}, 0xc0036f68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031d67740}, {0xc00102e4e0}, 0xc06b8bdb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031d676e0}, 0xc06b8bdb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0666fdb90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219185794 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa03c8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0cd1ee700, 0xc0cdc0d000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0cd1ee700, {0xc0cdc0d000, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0cd1ee700, {0xc0cdc0d000, 0x739, 0xc0cdc1a640}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0324308f8, {0xc0cdc0d000, 0x7eda79, 0xc0cdc0d005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0cd32b9e0, {0xc0cdc0d000, 0xc07e24ac60, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc05bebf668, {0xc0cdc0d000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0cd34c978, {0x631b900, 0xc05bebf668}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0cd34c700, {0x7fe42ab59310, 0xc0cd32b9e0}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0cd34c700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0cd34c700, {0xc099d47000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0cdc26840, {0xc099d47000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a546ab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a546ab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a546b0e0, {0x634aac0, 0xc0a546ab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a546aa80, 0xc054206b30, 0xc076d9342b, 0xc076d9342d, {0x635bec0, 0xc0cadf58c0}, 0xc000d6ae00, 0xc036570c48, 0xc076d93429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a546aa80, {0x63c85b8, 0xc0cdc3a690}, 0xc054206b20, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc045b88021, 0xb}}, 0x0, 0xc0cdc3a630, 0xc0cdc3a660, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0cdc3a690}, {0x643d810, 0xc0cd32b9e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0cd72c0f0}, {0x643d810, 0xc0cd32b9e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0cd72c0f0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805753 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc012027400, {0x63c85b8, 0xc0659ff530}, 0xc0b9445100, 0xc07a18ef00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc012027400, {0x63c85b8, 0xc0659ff530}, 0xc0b9445100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011047e175ba, 0x0, 0x0}, 0xc0b49a8350, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011047e175ba, 0x0, 0x0}, 0xc0b49a8350, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0659ff4a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0659ff4a0}, {0xc003df93f8, 0x660089}, 0xc003df9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0659ff4a0}, {0x0}, 0xc0b9444f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0659ff410}, 0xc0b9444f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}, 0xc007819b60, 0xc04cebae40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}, 0x3, 0xc03b158cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003df9980}}}, {0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}, 0xc007819b60, 0xc03b158cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0659ff410}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0659ff410}, {0x40, 0xc003df9b70}, 0xc003df9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0659ff410}, {0x4e9d7a0, 0xc0b9444f00}, 0xc007819b60, 0xc04cebae40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0659ff410}, 0xc03ffcd920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0937326c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0937326c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 216536881 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab978b0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04f345400, 0xc01f542000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04f345400, {0xc01f542000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04f345400, {0xc01f542000, 0x5, 0xc058f26bc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc022263748, {0xc01f542000, 0x80, 0x6}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01abd4ea0, {0xc01f542000, 0xc01b7e2c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc076fdbb48, {0xc01f542000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc097e3d078, {0x631b900, 0xc076fdbb48}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc097e3ce00, {0x7fe42ab59310, 0xc01abd4ea0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc097e3ce00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc097e3ce00, {0xc050bda000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00b73e4e0, {0xc050bda000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a943a088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a943a088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a943a660, {0x634aac0, 0xc0a943a088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a943a000, 0xc0580d9160, 0xc058f2742b, 0xc058f2742d, {0x635bec0, 0xc0497dcb40}, 0xc000d6ae00, 0xc06148f168, 0xc058f27429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a943a000, {0x63c85b8, 0xc022495bf0}, 0xc0580d9150, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc05777c025, 0x8}}, 0x0, 0xc022495b60, 0xc022495bc0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc022495bf0}, {0x643d810, 0xc01abd4ea0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03508ffb0}, {0x643d810, 0xc01abd4ea0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03508ffb0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803830 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01d898800, {0x63c85b8, 0xc07e1ba840}, 0xc097368000, 0xc03053b860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc07e1ba840}, 0xc097368000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032122250, 0x0, 0x0}, 0xc0a0050cc0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032122250, 0x0, 0x0}, 0xc0a0050cc0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07e1ba7b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e1ba7b0}, {0xc0743d73f8, 0x660089}, 0xc0743d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07e1ba7b0}, {0x7fe3f32c3070}, 0xc0613c3e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07e1ba720}, 0xc0613c3e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}, 0xc0606eb600, 0xc038313d58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}, 0x3, 0xc02661c680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0743d7980}}}, {0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}, 0xc0606eb600, 0xc02661c680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07e1ba720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e1ba720}, {0x40, 0xc0743d7b70}, 0xc0743d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07e1ba720}, {0x4e9d7a0, 0xc0613c3e00}, 0xc0606eb600, 0xc038313d58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07e1ba720}, 0xc0362925a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803909 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc05f7f7800}, 0xc0715c8018, 0x1, 0xc0708690a8, 0x519bbc, 0xc04afa96e0, 0xc04c2b4d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc05f7f7800}, 0x1e0, 0xc070869230, 0x4afa9680, 0x0, 0x660069, 0xc04afa96e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc05f7f7800}, 0xc04afa9680, {{{{0x0, 0x0}, {0xc071471440, 0xe6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc02dd741e0, {0x63c85b8, 0xc05f7f7800}, 0xc06d05ccf0, 0x70869428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc05f7f7800}, {0xc04abd99e0, {0x1743011030dac0b1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc05f7f7800}, 0xc053fbc640, 0xc0708699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05f7f7800}, 0x660089, {0xc04abd99e0, {0x1743011030dac0b1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc05f7f7800}, 0xc076b15500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011030dac0b1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011030dac0b1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05f7f7770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f7f7770}, {0xc07086b3f8, 0x660089}, 0xc07086b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05f7f7770}, {0xc0015221e0}, 0xc076b15300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05f7f76e0}, 0xc076b15300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}, 0xc0245903a0, 0xc0683c9c50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}, 0x3, 0xc038ec9440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07086b980}}}, {0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}, 0xc0245903a0, 0xc038ec9440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05f7f76e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f7f76e0}, {0x40, 0xc07086bb70}, 0xc07086bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05f7f76e0}, {0x4e9d7a0, 0xc076b15300}, 0xc0245903a0, 0xc0683c9c50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05f7f76e0}, 0xc042fe0120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04abd98c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04abd98c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219143270 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa9880, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07a38cc00, 0xc08a9f0000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07a38cc00, {0xc08a9f0000, 0xdaa8, 0xdaa8}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07a38cc00, {0xc08a9f0000, 0xbf, 0xc01e250708}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc022263b00, {0xc08a9f0000, 0x6, 0xc01e250788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc023bdf1e8, {0xc08a9f0000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0a05f0978, {0x631b900, 0xc023bdf1e8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0a05f0700, {0x63290e0, 0xc022263b00}, 0x2) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0a05f0700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0a05f0700, {0xc07c400000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc0427e37a0, {0xc03535c820, 0x9, 0xc065318bd0}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc0427e37a0}, {0xc03535c820, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc03535c820, 0x9, 0xc065318b10}, {0x631b600, 0xc0427e37a0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc03535c7e0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc019e65e00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 219803783 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc07ac27590}, {0xc04be26000, 0x6, 0xc04fb1de58}, {0x0, {0x1743010ff048ae10, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc07ac27590}, 0x12dcd, 0xc06565f320, 0x20, 0xc081e53be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc07ac27590}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808382 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc001639900, {0x63c85b8, 0xc02db44b10}, 0xc02b96be48, 0x1, 0xc00602d0a8, 0x519bbc, 0xc0621a7140, 0xc079914180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02db44b10}, 0x120, 0xc00602d230, 0x621a70e0, 0x0, 0x660069, 0xc0621a7140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc001639900, {0x63c85b8, 0xc02db44b10}, 0xc0621a70e0, {{{{0x0, 0x0}, {0xc073b63560, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02db44b10}, 0x6352410, 0x3f8ea5b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00aa54af0, {0x63c85b8, 0xc02db44b10}, {0xc0621a6fc0, {0x174301105b709713, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00aa54af0, {0x63c85b8, 0xc02db44b10}, 0xc02e903220, 0xc00602d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02db44b10}, 0x660089, {0xc0621a6fc0, {0x174301105b709713, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001ae9000, {0x63c85b8, 0xc02db44b10}, 0xc08993ad00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b709713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b709713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04cccbf20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cccbf20}, {0xc00602f3f8, 0x660089}, 0xc00602f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04cccbf20}, {0x3feffffee1276011}, 0xc08993ab00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04cccbe90}, 0xc08993ab00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}, 0xc061a4e460, 0xc06d54a750) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}, 0x3, 0xc0100dcd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00602f980}}}, {0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}, 0xc061a4e460, 0xc0100dcd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04cccbe90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cccbe90}, {0x40, 0xc00602fb70}, 0xc00602fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04cccbe90}, {0x4e9d7a0, 0xc08993ab00}, 0xc061a4e460, 0xc06d54a750) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04cccbe90}, 0xc08ce8b500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0621a6d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0621a6d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809031 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc02dd6fa10}, 0xc04a569800, 0xc0797d50e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc02dd6fa10}, 0xc04a569800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110635c4cbc, 0x0, 0x0}, 0xc04023fbe0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110635c4cbc, 0x0, 0x0}, 0xc04023fbe0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02dd6f890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02dd6f890}, {0xc07ce8d2a8, 0x660089}, 0xc07ce8d2b0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02dd6f890}, {0xc00102e4e0}, 0xc04a569700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02dd6f830}, 0xc04a569700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8580, 0xc078843740}, 0x0, 0xc06606f080) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c8580, 0xc078843740}, 0x63c8580) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftLogQueue).process(0xc003629840, {0x63c8580, 0xc078843740}, 0xc05b267000, {0xc078843740, 0x65f500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_log_queue.go:740 +0x87c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica.func1({0x63c8580, 0xc078843740}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:977 +0x26e github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02dd6f650}, {0xc0600c0750, 0x24}, 0xdf8475800, 0xc002fede28) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica(0xc00d506f00, {0x63c85b8, 0xc02dd6f530}, {0x6463f50, 0xc05b267000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:936 +0x3f5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2.1({0x63c85b8, 0xc02dd6f530}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:855 +0xde github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219719081 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0af90c160, {0x63c85b8, 0xc054707830}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc065fff300, {0x63c85b8, 0xc054707830}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc065fff300, {0x63c8510, 0xc074eacfc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc065fff300, {0x63c8510, 0xc074eacfc0}, 0xc044ff3730, {0x7fe42b790cf0, 0xc066c03bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc065fff300, {0x63c85b8, 0xc04f1a6690}, {{0x63f88f8, 0xc044ff3730}, {0xc07059cc8d, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc065fff300, {0x63c85b8, 0xc04f1a6690}, {{0x63f88f8, 0xc044ff3730}, {0xc07059cc8d, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc044ff3730}, {0xc07059cc8d, 0x23}, 0x0, 0x0}, {0x2f6085f, 0xedb7a8350, 0x0}, {0x2f6111e, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc065fff300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc065fff300, {0x63c8510, 0xc016176740}, 0xc06010ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc042f18ad0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc016176740}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc042f18ad0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 28275 [select, 225 minutes]: github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging.(*CaptureIndexUsageStatsLoggingScheduler).start.func1({0x63c85b8, 0xc02ce14a20}) github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging/captured_index_usage_stats.go:144 +0x105 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 17682 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219805090 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc06fcdb590}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 214445169 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc05815a590, 0x1a0) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc05457ea80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc05457f0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc057060600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc057060600, {0x63c8510, 0xc05815a5c0}, 0xc055cc7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07b30ee00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc05815a5c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07b30ee00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812818 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e78ac80, {0x63c85b8, 0xc05c285c80}, 0xc0c884d6e0, 0x1, 0xc002a1d0a8, 0x519bbc, 0xc0117adb60, 0xc039fabc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05c285c80}, 0x120, 0xc002a1d230, 0x117adb00, 0x0, 0x660069, 0xc0117adb60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e78ac80, {0x63c85b8, 0xc05c285c80}, 0xc0117adb00, {{{{0x0, 0x0}, {0xc042834fc0, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05c285c80}, 0x6352410, 0x3c90c598) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0180e3bd0, {0x63c85b8, 0xc05c285c80}, {0xc0117ad8c0, {0x174301107f056b0c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0180e3bd0, {0x63c85b8, 0xc05c285c80}, 0xc05292edc0, 0xc002a1d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05c285c80}, 0x660089, {0xc0117ad8c0, {0x174301107f056b0c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e7d3000, {0x63c85b8, 0xc05c285c80}, 0xc02664f900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f056b0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f056b0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05c285bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c285bf0}, {0xc002a1f3f8, 0x660089}, 0xc002a1f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05c285bf0}, {0x7fdd4a372b00}, 0xc02664f700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05c285b60}, 0xc02664f700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}, 0xc03d9ad740, 0xc023415cc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}, 0x3, 0xc048fdec80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002a1f980}}}, {0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}, 0xc03d9ad740, 0xc048fdec80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05c285b60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c285b60}, {0x40, 0xc002a1fb70}, 0xc002a1fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05c285b60}, {0x4e9d7a0, 0xc02664f700}, 0xc03d9ad740, 0xc023415cc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05c285b60}, 0xc092bf2840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0117ac6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0117ac6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807796 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc077f01ce0}, 0xc03a23dec0, 0x1, 0xc0030930a8, 0x519bbc, 0xc035c04600, 0xc035c04420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc077f01ce0}, 0x1e0, 0xc003093230, 0x35c045a0, 0x0, 0x660069, 0xc035c04600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc077f01ce0}, 0xc035c045a0, {{{{0x0, 0x0}, {0xc093c1efc0, 0x25}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc077f01ce0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc077f01ce0}, {0xc0741e9200, {0x174301105a326828, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc077f01ce0}, 0xc05539f540, 0xc0030939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077f01ce0}, 0x660089, {0xc0741e9200, {0x174301105a326828, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc077f01ce0}, 0xc01c758e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a326828, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a326828, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077f01bc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077f01bc0}, {0xc0030953f8, 0x660089}, 0xc003095400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077f01bc0}, {0x3feffffffd645302}, 0xc01c758900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077f01b30}, 0xc01c758900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}, 0xc06be61000, 0xc023c490f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}, 0x3, 0xc0830ce500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003095980}}}, {0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}, 0xc06be61000, 0xc0830ce500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc077f01b30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077f01b30}, {0x40, 0xc003095b70}, 0xc003095ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc077f01b30}, {0x4e9d7a0, 0xc01c758900}, 0xc06be61000, 0xc023c490f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc077f01b30}, 0xc08db86960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0783fb200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0783fb200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812317 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0707b4030}, 0xc0403e7a40, 0x1, 0xc0072f16d8, 0x519bbc, 0xc00b832660, 0xc052f949c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc037dc1560, {0x63c85b8, 0xc0707b4030}, 0xc0cd8fd9e0, 0xc0072f17d8, 0xb832660, 0x0, 0x0, 0xc00b832660, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0707b4030}, 0xc00b8325a0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc06de018c0, 0xaf}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0707b4030}, 0x6352260, 0x655a2728) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0707b4030}, {0xc00b832000, {0x174301104cf73160, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0707b4030}, 0xc0c91208c0, 0xc0072f2020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0707b4030}, 0x660089, {0xc00b832000, {0x174301104cf73160, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0707b4030}, 0xc0a5abfc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104cf73160, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104cf73160, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07d327f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07d327f80}, {0xc07c881a28, 0x660089}, 0xc07c881a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07d327f80}, {0xc00102e4e0}, 0xc0a5abfb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07d327f20}, 0xc0a5abfb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc04d0657d0, {0x63c85b8, 0xc07d327e90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc04d0657d0, {0x63c85b8, 0xc07d327e90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc07d327e90}, {0x4f67ad2, 0x6446ff8}, 0xc002d9cfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 4697 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00af3da90, {0xc016613378, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00af3da90, {0xc016613378, 0xc0197f7de8, 0xc002ca2a68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc01d4ca9f0, {0xc016613378, 0xc002ca2ae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc01d4ca9f0}, {0xc016613378, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc01300f440, {0xc016613378, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc016613368, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x0, 0xc01300f440, {0x0, 0x0}, 0x7fffffff, 0xc002ca2d90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0x0, {0x0, 0x0}, {0x4cfe5a0, 0xc02902db18}, 0x1d0000001d, 0xc002ca2d90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc0016e32b0, {0x4cfe5a0, 0xc02902db18}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0153e3ac0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc0153e3ac0}, 0xc0023423b0, {0x63c85b8, 0xc01d4cac00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc01d4cac00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810212 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021a5f2c0, {0x63c85b8, 0xc01ca8c360}, 0xc0152196b0, 0x1, 0xc01a0b70a8, 0x519bbc, 0xc0529d2940, 0xc036843ce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05a73c480, {0x63c85b8, 0xc01ca8c360}, 0xc05a73c4b0, 0xc01a0b71a8, 0x30, 0x40, 0xffffffffffffffff, 0xc0529d2940, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021a5f2c0, {0x63c85b8, 0xc01ca8c360}, 0xc0529d2700, {{{{0x0, 0x0}, {0xc060fc7320, 0x3}}, {{0xc068123200, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01ca8c360}, 0x63521a0, 0x7818e5a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a30d20, {0x63c85b8, 0xc01ca8c360}, {0xc04721a000, {0x1743010ff087dec8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a30d20, {0x63c85b8, 0xc01ca8c360}, 0xc0d051aaa0, 0xc01a0b79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01ca8c360}, 0x660089, {0xc04721a000, {0x1743010ff087dec8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b1000, {0x63c85b8, 0xc01ca8c360}, 0xc032cf6800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff087dec8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff087dec8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01ca8c2d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ca8c2d0}, {0xc01a0b93f8, 0x660089}, 0xc01a0b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01ca8c2d0}, {0x7fe3c0192958}, 0xc032cf6600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01ca8c210}, 0xc032cf6600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}, 0xc07cd09520, 0xc02532c300) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}, 0x3, 0xc04517f840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a0b9980}}}, {0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}, 0xc07cd09520, 0xc04517f840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01ca8c210}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ca8c210}, {0x40, 0xc01a0b9b70}, 0xc01a0b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01ca8c210}, {0x4e9d7a0, 0xc032cf6600}, 0xc07cd09520, 0xc02532c300) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01ca8c210}, 0xc016525440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06d26e120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06d26e120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28065 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc007d03290, {0x63c8510, 0xc00abaa8c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:319 +0x997 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x63c85b8, 0xc02cafa030}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:189 +0x274 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219149965 [select, 1 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0120dac60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc028bc2790, {0x4cfe6e0, 0xc053034c00}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc021b91560, 0xc001b0edb0, 0xc001b0ed80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc021b91560, {0x4cfe6e0, 0xc053034c00}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc07487fc00, {0x4cfe6e0, 0xc053034c00}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc02c02bb50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc02c02bb50}, 0xc0181fa800, 0xc000cb3540, {0x63c85b8, 0xc01bf83710}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc01bf83710}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 4224 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00af3d900, {0xc015d8a340, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00af3d900, {0xc015d8a340, 0xc0197f6c60, 0xc002ca1a68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc01cc5dce0, {0xc015d8a340, 0xc002ca1ae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc01cc5dce0}, {0xc015d8a340, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0124e3320, {0xc015d8a340, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc015d8a330, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x0, 0xc0124e3320, {0x0, 0x0}, 0x7fffffff, 0xc002ca1d90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0x0, {0x0, 0x0}, {0x4cfe5a0, 0xc060f8e2a0}, 0x1a0000001b, 0xc002ca1d90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc0016e31e0, {0x4cfe5a0, 0xc060f8e2a0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0153e2ee0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc0153e2ee0}, 0xc0315fe9b0, {0x63c85b8, 0xc01d030210}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc01d030210}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805121 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc060d17ce0}, 0xc036f97320, 0x1, 0x0, 0x1, 0xc066e0f5c0, 0xc0510b5260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc060d17ce0}, 0x120, 0xc095f6a728, 0x66e0f560, 0xc00c1c9d40, 0x18, 0xc066e0f5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc060d17ce0}, 0xc066e0f560, {{{{0x0, 0x0}, {0xc05748cb40, 0x22}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc060d17ce0}, 0x6352410, 0x1f8e1640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc060d17ce0}, {0xc0722aea20, {0x174301101b746c81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc060d17ce0}, 0xc02aca4f00, 0xc08da96ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc060d17ce0}, 0x660089, {0xc0722aea20, {0x174301101b746c81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc060d17ce0}, 0xc06c4b6b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b746c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b746c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060d17c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060d17c50}, {0xc0936928f0, 0x660089}, 0xc0936928f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060d17c50}, {0xc00102e4e0}, 0xc06c4b6900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc060d17bf0}, 0xc06c4b6900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06c2f9380}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808426 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4def8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc027152900, 0xc0143c0800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc027152900, {0xc0143c0800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc027152900, {0xc0143c0800, 0x739, 0xc089cc3a40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01b929c70, {0xc0143c0800, 0x7eda79, 0xc0143c0805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc07c5dda40, {0xc0143c0800, 0x2, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc044e34348, {0xc0143c0800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03ee34978, {0x631b900, 0xc044e34348}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03ee34700, {0x7fe42ab59310, 0xc07c5dda40}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03ee34700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03ee34700, {0xc07554d000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0492ae348, {0xc07554d000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc033763588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc033763588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc033763b60, {0x634aac0, 0xc033763588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc033763500, 0xc04325ede0, 0xc00a86142b, 0xc00a86142d, {0x635bec0, 0xc00f90c090}, 0xc000d6ae00, 0xc05c9b7b54, 0xc00a861429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc033763500, {0x63c85b8, 0xc06d2b2f30}, 0xc04325edd0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a5eaf043, 0xb}}, 0x0, 0xc06d2b2ed0, 0xc06d2b2f00, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06d2b2f30}, {0x643d810, 0xc07c5dda40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0200f42d0}, {0x643d810, 0xc07c5dda40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0200f42d0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808383 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03bd19180, {0x63c85b8, 0xc0571c1bf0}, 0xc03b644c18, 0x1, 0xc0104b70a8, 0x519bbc, 0xc06fba9140, 0xc035c058c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0571c1bf0}, 0x1e0, 0xc0104b7230, 0x6fba90e0, 0x0, 0x660069, 0xc06fba9140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03bd19180, {0x63c85b8, 0xc0571c1bf0}, 0xc06fba90e0, {{{{0x0, 0x0}, {0xc059633560, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc0a5938cc0, {0x63c85b8, 0xc0571c1bf0}, 0xc081deb530, 0x104b7428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc067985680, {0x63c85b8, 0xc0571c1bf0}, {0xc07af69200, {0x174301105b709713, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc067985680, {0x63c85b8, 0xc0571c1bf0}, 0xc02c214f00, 0xc0104b79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0571c1bf0}, 0x660089, {0xc07af69200, {0x174301105b709713, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc0571c1bf0}, 0xc05bc3fb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b709713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b709713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0571c1b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0571c1b60}, {0xc0104b93f8, 0x660089}, 0xc0104b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0571c1b60}, {0x7fe423edbcb0}, 0xc05bc3f900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0571c1ad0}, 0xc05bc3f900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}, 0xc0476476c0, 0xc07e1177b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}, 0x3, 0xc01a45ccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0104b9980}}}, {0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}, 0xc0476476c0, 0xc01a45ccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0571c1ad0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0571c1ad0}, {0x40, 0xc0104b9b70}, 0xc0104b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0571c1ad0}, {0x4e9d7a0, 0xc05bc3f900}, 0xc0476476c0, 0xc07e1177b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0571c1ad0}, 0xc0286c21e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0621a6ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0621a6ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811617 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc06b42d680}, 0xc03b040e70, 0x1, 0x7080181ac0e418, 0x37e11d600, 0xc046a31140, 0xc059ec8600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06b42d680}, 0x1e0, 0xc0376f7230, 0x46a310e0, 0x0, 0x660069, 0xc046a31140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc06b42d680}, 0xc046a310e0, {{{{0x0, 0x0}, {0xc06e98c900, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06b42d680}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc06b42d680}, {0xc044222b40, {0x174301107dc2e172, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc06b42d680}, 0xc0205a2140, 0xc0376f79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b42d680}, 0x660089, {0xc044222b40, {0x174301107dc2e172, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc06b42d680}, 0xc05bd2d700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107dc2e172, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107dc2e172, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b42d5f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b42d5f0}, {0xc0376f93f8, 0x660089}, 0xc0376f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b42d5f0}, {0x7fe3b6700b20}, 0xc05bd2d500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b42d560}, 0xc05bd2d500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}, 0xc01e516ba0, 0xc03b040e40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}, 0x3, 0xc018742dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0376f9980}}}, {0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}, 0xc01e516ba0, 0xc018742dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b42d560}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b42d560}, {0x40, 0xc0376f9b70}, 0xc0376f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b42d560}, {0x4e9d7a0, 0xc05bd2d500}, 0xc01e516ba0, 0xc03b040e40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b42d560}, 0xc08c159e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0442226c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0442226c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28466 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00145d600, {0x63c8510, 0xc01e61bac0}, 0xc02cf44bd0, 0xc01d962060) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:331 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x63c8510, 0xc01e61bac0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:288 +0x38 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219803574 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04e0de420}, 0xc026787f20, 0x1, 0xc076c370a8, 0x519bbc, 0xc048e3c9c0, 0xc048f18d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04e0de420}, 0x1e0, 0xc076c37230, 0x48e3c960, 0x0, 0x660069, 0xc048e3c9c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04e0de420}, 0xc048e3c960, {{{{0x0, 0x0}, {0xc052fff8c0, 0x44}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04e0de420}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04e0de420}, {0xc03e7e30e0, {0x174301102e330700, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04e0de420}, 0xc068090960, 0xc076c379f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e0de420}, 0x660089, {0xc03e7e30e0, {0x174301102e330700, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04e0de420}, 0xc05ee2ac00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e330700, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e330700, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e0de390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e0de390}, {0xc076c393f8, 0x660089}, 0xc076c39400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e0de390}, {0x0}, 0xc05ee2aa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e0de300}, 0xc05ee2aa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}, 0xc04547dda0, 0xc0725500d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}, 0x3, 0xc039150580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076c39980}}}, {0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}, 0xc04547dda0, 0xc039150580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e0de300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e0de300}, {0x40, 0xc076c39b70}, 0xc076c39ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e0de300}, {0x4e9d7a0, 0xc05ee2aa00}, 0xc04547dda0, 0xc0725500d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e0de300}, 0xc06c7b5680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03e7e2fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03e7e2fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219790472 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafeea0f0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0a0355600, 0xc074c79000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0a0355600, {0xc074c79000, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0a0355600, {0xc074c79000, 0x0, 0x4d889e}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03102fec8, {0xc074c79000, 0x18b71da, 0xc07af8c900}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc039116a20, {0xc074c79000, 0x1, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc021b9b128, {0xc074c79000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0532c3778, {0x631b900, 0xc021b9b128}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0532c3500, {0x7fe42ab59310, 0xc039116a20}, 0x63c85b8) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0532c3500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0532c3500, {0xc08f454000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0361346f0, {0xc08f454000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03949e088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03949e088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03949e660, {0x634aac0, 0xc03949e088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03949e000, 0xc016087d50, 0xc07265942b, 0xc07265942d, {0x635bec0, 0xc013f84ab0}, 0xc000d6ae00, 0xc036a88664, 0xc072659429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03949e000, {0x63c85b8, 0xc043352d20}, 0xc016087d40, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc096e30043, 0xb}}, 0x0, 0xc043352cc0, 0xc043352cf0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc043352d20}, {0x643d810, 0xc039116a20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc071c0dd40}, {0x643d810, 0xc039116a20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc071c0dd40}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28327 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0003f41e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 219802758 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafeea2d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc066c4f480, 0xc03ccdf800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc066c4f480, {0xc03ccdf800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc066c4f480, {0xc03ccdf800, 0x735, 0xc03ac33680}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0473dc9a0, {0xc03ccdf800, 0x7eda79, 0xc03ccdf805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0292dc7e0, {0xc03ccdf800, 0x6, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc021b9b1a0, {0xc03ccdf800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03bfcbaf8, {0x631b900, 0xc021b9b1a0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03bfcb880, {0x7fe42ab59310, 0xc0292dc7e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03bfcb880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03bfcb880, {0xc06545a000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc059df8ff0, {0xc06545a000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc044aa2088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc044aa2088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc044aa2660, {0x634aac0, 0xc044aa2088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc044aa2000, 0xc04038c670, 0xc002d5d42b, 0xc002d5d42d, {0x635bec0, 0xc076d0d830}, 0xc000d6ae00, 0xc02c3c8f08, 0xc002d5d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc044aa2000, {0x63c85b8, 0xc03c765050}, 0xc04038c660, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc01ab10047, 0x8}}, 0x0, 0xc03c764fc0, 0xc03c764ff0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03c765050}, {0x643d810, 0xc0292dc7e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc050360bd0}, {0x643d810, 0xc0292dc7e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc050360bd0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802670 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02c4a8c00, {0x63c85b8, 0xc047e33350}, 0xc046fdd200, 0xc06b233ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc047e33350}, 0xc046fdd200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110256452d9, 0x0, 0x0}, 0xc03060b090, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110256452d9, 0x0, 0x0}, 0xc03060b090, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e332c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e332c0}, {0xc064f333f8, 0x660089}, 0xc064f33400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e332c0}, {0x7fe3b7a80c30}, 0xc046fdd000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e33230}, 0xc046fdd000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}, 0xc0968d0840, 0xc06316fd70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}, 0x3, 0xc056d17980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc064f33980}}}, {0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}, 0xc0968d0840, 0xc056d17980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047e33230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e33230}, {0x40, 0xc064f33b70}, 0xc064f33ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047e33230}, {0x4e9d7a0, 0xc046fdd000}, 0xc0968d0840, 0xc06316fd70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047e33230}, 0xc0974e2a20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0bcf73b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0bcf73b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809199 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc058fc0360}, 0xc032a1f300, 0xc079425ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc058fc0360}, 0xc032a1f300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011064181c29, 0x0, 0x0}, 0xc056bfe430, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011064181c29, 0x0, 0x0}, 0xc056bfe430, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc058fc02d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058fc02d0}, {0xc0127f53f8, 0x660089}, 0xc0127f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc058fc02d0}, {0x0}, 0xc032a1ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc058fc0240}, 0xc032a1ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}, 0xc02fa350c0, 0xc01e9be018) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}, 0x3, 0xc0525cd000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0127f5980}}}, {0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}, 0xc02fa350c0, 0xc0525cd000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc058fc0240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058fc0240}, {0x40, 0xc0127f5b06}, 0xc0127f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc058fc0240}, {0x4e9d7a0, 0xc032a1ee00}, 0xc02fa350c0, 0xc01e9be018) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc058fc0240}, 0xc022852ea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052b79e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052b79e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28234 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x63c85b8, 0xc02cc8b9e0}) github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:268 +0x32e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808062 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021cd2000, {0x63c85b8, 0xc051ff9c50}, 0xc02c5ecab0, 0x1, 0xc0054d90a8, 0x519bbc, 0xc01978e2a0, 0xc079b024e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc051ff9c50}, 0x120, 0xc0054d9230, 0x1978e240, 0x0, 0x660069, 0xc01978e2a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021cd2000, {0x63c85b8, 0xc051ff9c50}, 0xc01978e240, {{{{0x0, 0x0}, {0xc0576c6d80, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc051ff9c50}, 0x6352410, 0x66762d50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c432c0, {0x63c85b8, 0xc051ff9c50}, {0xc07a90d9e0, {0x174301105a865aee, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c432c0, {0x63c85b8, 0xc051ff9c50}, 0xc0632610e0, 0xc0054d99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc051ff9c50}, 0x660089, {0xc07a90d9e0, {0x174301105a865aee, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd6000, {0x63c85b8, 0xc051ff9c50}, 0xc02a7d7500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a865aee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a865aee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059f82420}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059f82420}, {0xc0054db3f8, 0x660089}, 0xc0054db400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059f82420}, {0x7fe3bc1b6960}, 0xc0452da300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc059f82390}, 0xc0452da300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}, 0xc02ce7f860, 0xc0293d8750) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}, 0x3, 0xc053d7b600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0054db980}}}, {0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}, 0xc02ce7f860, 0xc053d7b600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc059f82390}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059f82390}, {0x40, 0xc0054dbb70}, 0xc0054dbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc059f82390}, {0x4e9d7a0, 0xc0452da300}, 0xc02ce7f860, 0xc0293d8750) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc059f82390}, 0xc0327d1860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc079ab0900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc079ab0900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810409 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc025c3f000, {0x63c85b8, 0xc04cd0aab0}, 0xc0d233f200, 0xc065740f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025c3f000, {0x63c85b8, 0xc04cd0aab0}, 0xc0d233f200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f793c8b, 0x0, 0x0}, 0xc04c35c900, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f793c8b, 0x0, 0x0}, 0xc04c35c900, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04cd0a9f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cd0a9f0}, {0xc01baf73f8, 0x660089}, 0xc01baf7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04cd0a9f0}, {0x0}, 0xc0d233f000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04cd0a960}, 0xc0d233f000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}, 0xc020f95800, 0xc04917c648) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}, 0x3, 0xc0c3cb9300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01baf7980}}}, {0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}, 0xc020f95800, 0xc0c3cb9300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04cd0a960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cd0a960}, {0x40, 0xc01baf7b70}, 0xc01baf7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04cd0a960}, {0x4e9d7a0, 0xc0d233f000}, 0xc020f95800, 0xc04917c648) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04cd0a960}, 0xc0729b1740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc054b297a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc054b297a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811743 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a2ad540, {0x63c85b8, 0xc0438f29c0}, 0xc05796b848, 0x1, 0xc0322f30a8, 0x519bbc, 0xc073206420, 0xc05311dc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0438f29c0}, 0x1e0, 0xc0322f3230, 0x732063c0, 0x0, 0x660069, 0xc073206420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a2ad540, {0x63c85b8, 0xc0438f29c0}, 0xc0732063c0, {{{{0x0, 0x0}, {0xc0468c5680, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc0253807e0, {0x63c85b8, 0xc0438f29c0}, 0xc0036c3270, 0x322f3428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a292e10, {0x63c85b8, 0xc0438f29c0}, {0xc017662fc0, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a292e10, {0x63c85b8, 0xc0438f29c0}, 0xc075884460, 0xc0322f39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0438f29c0}, 0x660089, {0xc017662fc0, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a2f6c00, {0x63c85b8, 0xc0438f29c0}, 0xc059d4ca00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0438f2930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0438f2930}, {0xc0322f53f8, 0x660089}, 0xc0322f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0438f2930}, {0x0}, 0xc059d4c800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0438f28a0}, 0xc059d4c800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}, 0xc023d93be0, 0xc0723264f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}, 0x3, 0xc041ef74c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0322f5980}}}, {0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}, 0xc023d93be0, 0xc041ef74c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0438f28a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0438f28a0}, {0x40, 0xc0322f5b70}, 0xc0322f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0438f28a0}, {0x4e9d7a0, 0xc059d4c800}, 0xc023d93be0, 0xc0723264f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0438f28a0}, 0xc0673bde60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc017662c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc017662c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805715 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc049ad0a50}, 0xc03c3a2588, 0x1, 0x708017e363d19b, 0x37e11d600, 0xc0518c6600, 0xc05c9798c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc049ad0a50}, 0x1e0, 0xc08d5bb230, 0x518c65a0, 0xc01faeab60, 0x18, 0xc0518c6600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc049ad0a50}, 0xc0518c65a0, {{{{0x0, 0x0}, {0xc060018000, 0x58}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc049ad0a50}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc049ad0a50}, {0xc063c1b560, {0x174301104555bd36, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc049ad0a50}, 0xc072b91e00, 0xc08d5bb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc049ad0a50}, 0x660089, {0xc063c1b560, {0x174301104555bd36, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc049ad0a50}, 0xc059c67300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104555bd36, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104555bd36, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc049ad09c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049ad09c0}, {0xc08d5bd3f8, 0x660089}, 0xc08d5bd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc049ad09c0}, {0xc0015221e0}, 0xc059c67000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc049ad0930}, 0xc059c67000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}, 0xc01e670660, 0xc03c3a24f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}, 0x3, 0xc08214e180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc08d5bd980}}}, {0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}, 0xc01e670660, 0xc08214e180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc049ad0930}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049ad0930}, {0x40, 0xc08d5bdb70}, 0xc08d5bdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}, 0x203020, 0x203020) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc049ad0930}, {0x4e9d7a0, 0xc059c67000}, 0xc01e670660, 0xc03c3a24f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc049ad0930}, 0xc038828360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060690fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060690fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811249 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021deec80, {0x63c85b8, 0xc0473c5170}, 0xc06292f398, 0x1, 0x0, 0x1, 0xc0308d6840, 0xc072dbfda0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0473c5170}, 0x120, 0xc02eb10728, 0x308d67e0, 0x0, 0x660069, 0xc0308d6840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021deec80, {0x63c85b8, 0xc0473c5170}, 0xc0308d67e0, {{{{0x0, 0x0}, {0xc03aaf4480, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0473c5170}, 0x6352410, 0x47a9c8b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3d860, {0x63c85b8, 0xc0473c5170}, {0xc061db58c0, {0x17430110713c74c9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3d860, {0x63c85b8, 0xc0473c5170}, 0xc051998140, 0xc089d54ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0473c5170}, 0x660089, {0xc061db58c0, {0x17430110713c74c9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021e97000, {0x63c85b8, 0xc0473c5170}, 0xc045e94f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110713c74c9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110713c74c9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0473c50e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473c50e0}, {0xc0936968f0, 0x660089}, 0xc0936968f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0473c50e0}, {0xc00102e4e0}, 0xc045e94e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0473c5080}, 0xc045e94e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc05f77df80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807396 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00036a800, {0x63c85b8, 0xc0437002a0}, 0xc0913cc800, 0xc043044fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc0437002a0}, 0xc0913cc800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011056dbf91e, 0x0, 0x0}, 0xc08c88f190, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011056dbf91e, 0x0, 0x0}, 0xc08c88f190, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043700210}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043700210}, {0xc005be3a08, 0x660089}, 0xc005be3a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043700210}, {0xc00102e4e0}, 0xc0913cc700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0437001b0}, 0xc0913cc700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc04b5a8c90}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04b5a8c90}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810001 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc047a42bd0}, 0xc024020108, 0x1, 0xc014c4f6d8, 0x519bbc, 0xc08224cba0, 0xc07a90ca80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc02fb002a0, {0x63c85b8, 0xc047a42bd0}, 0xc059b9a000, 0xc014c4f860, 0x8224cc00, 0x0, 0x0, 0xc08224cba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc047a42bd0}, 0xc08224cb40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0360a1680, 0x8f}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047a42bd0}, 0x6352260, 0x7b228498) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc047a42bd0}, {0xc08224ca20, {0x174301102f46acf5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc047a42bd0}, 0xc05953f220, 0xc014c50020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047a42bd0}, 0x660089, {0xc08224ca20, {0x174301102f46acf5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc047a42bd0}, 0xc0b31c1d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102f46acf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102f46acf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047a42b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047a42b10}, {0xc088137a28, 0x660089}, 0xc088137a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047a42b10}, {0xc00102e4e0}, 0xc0b31c1c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047a42a80}, 0xc0b31c1c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc058398cd0, {0x63c85b8, 0xc047a42840}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc058398cd0, {0x63c85b8, 0xc047a42840}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc047a42840}, {0x4f67ad2, 0x6446ff8}, 0xc088195fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219811897 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc054449f80}, 0xc063faecd8, 0x1, 0xc00fc910a8, 0x519bbc, 0xc04cabaa80, 0xc040c2ade0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc054449f80}, 0x120, 0xc00fc91230, 0x4cabaa20, 0x0, 0x660069, 0xc04cabaa80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc054449f80}, 0xc04cabaa20, {{{{0x0, 0x0}, {0xc0d08af0e0, 0x12}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc054449f80}, 0x6352410, 0x6b0667d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc054449f80}, {0xc04caba7e0, {0x174301107cc7b567, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc054449f80}, 0xc046dedae0, 0xc00fc919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc054449f80}, 0x660089, {0xc04caba7e0, {0x174301107cc7b567, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc054449f80}, 0xc0d29ece00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107cc7b567, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107cc7b567, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc054449ef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054449ef0}, {0xc00fc933f8, 0x660089}, 0xc00fc93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc054449ef0}, {0x7fe394579f98}, 0xc0d29ecc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc054449e60}, 0xc0d29ecc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}, 0xc08d07bce0, 0xc063faf4a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}, 0x3, 0xc05426c040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00fc93980}}}, {0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}, 0xc08d07bce0, 0xc05426c040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc054449e60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054449e60}, {0x40, 0xc00fc93b70}, 0xc00fc93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc054449e60}, {0x4e9d7a0, 0xc0d29ecc00}, 0xc08d07bce0, 0xc063faf4a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc054449e60}, 0xc05acd3aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04caba6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04caba6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28274 [select]: github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1({0x63c85b8, 0xc02ce149f0}) github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:443 +0x29f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803732 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0209ba000, {0x63c85b8, 0xc077722360}, 0xc04366e900, 0xc05b113720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209ba000, {0x63c85b8, 0xc077722360}, 0xc04366e900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ea3a0dc, 0x0, 0x0}, 0xc046af1a50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ea3a0dc, 0x0, 0x0}, 0xc046af1a50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0777222d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0777222d0}, {0xc076857918, 0x660089}, 0xc076857920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0777222d0}, {0xc00102e4e0}, 0xc04366e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077722270}, 0xc04366e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc03a6b9a40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0777221e0}, {0xc00dfd36b0, 0x24}, 0x46c7c147d, 0xc0385e7d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc0568f6360}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0777221e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219784866 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea3130, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07c1aa600, 0xc082bea000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07c1aa600, {0xc082bea000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07c1aa600, {0xc082bea000, 0x735, 0xc077ba7e00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc041cbacd8, {0xc082bea000, 0x7eda79, 0xc082bea005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc09c5446c0, {0xc082bea000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc044e349a8, {0xc082bea000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0797c8cf8, {0x631b900, 0xc044e349a8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0797c8a80, {0x7fe42ab59310, 0xc09c5446c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0797c8a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0797c8a80, {0xc043607000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc07d4470b0, {0xc043607000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc062bb8088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc062bb8088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc062bb8660, {0x634aac0, 0xc062bb8088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc062bb8000, 0xc07d45a8a0, 0xc019f1742b, 0xc019f1742d, {0x635bec0, 0xc07a7c1290}, 0xc000d6ae00, 0xc07a7a9b08, 0xc019f17429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc062bb8000, {0x63c85b8, 0xc081f9db30}, 0xc07d45a890, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc028e4c037, 0x8}}, 0x0, 0xc081f9dad0, 0xc081f9db00, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc081f9db30}, {0x643d810, 0xc09c5446c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07bf2d830}, {0x643d810, 0xc09c5446c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07bf2d830}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802458 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc061dce000, {0x63c85b8, 0xc0c78b4d80}, 0xc0a7740900, 0xc033dbad20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc0c78b4d80}, 0xc0a7740900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102367c3c2, 0x0, 0x0}, 0xc04fc368b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102367c3c2, 0x0, 0x0}, 0xc04fc368b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c78b4c60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c78b4c60}, {0xc073095918, 0x660089}, 0xc073095920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c78b4c60}, {0xc00102e4e0}, 0xc0a7740800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c78b4c00}, 0xc0a7740800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc05c82b6e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0c78b4b40}, {0xc00dfd36b0, 0x24}, 0x49876b7eb, 0xc016472d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc0d3a35e38, 0xc0781447c8}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0c78b4b40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809407 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03c908c80, {0x63c85b8, 0xc02fd9ba70}, 0xc029d0aa50, 0x1, 0xc010cab0a8, 0x519bbc, 0xc086e53920, 0xc05baef5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02fd9ba70}, 0x120, 0xc010cab230, 0x86e538c0, 0x0, 0x660069, 0xc086e53920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03c908c80, {0x63c85b8, 0xc02fd9ba70}, 0xc086e538c0, {{{{0x0, 0x0}, {0xc06870b320, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02fd9ba70}, 0x6352410, 0x6bab3e08) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00852fe50, {0x63c85b8, 0xc02fd9ba70}, {0xc086e537a0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00852fe50, {0x63c85b8, 0xc02fd9ba70}, 0xc075025900, 0xc010cab9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02fd9ba70}, 0x660089, {0xc086e537a0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b078c00, {0x63c85b8, 0xc02fd9ba70}, 0xc05b815300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02fd9b9e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fd9b9e0}, {0xc010cad3f8, 0x660089}, 0xc010cad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02fd9b9e0}, {0xc060bfa4a0}, 0xc05b815100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02fd9b860}, 0xc05b815100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}, 0xc03acd4760, 0xc08b141b78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}, 0x3, 0xc075c97700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010cad980}}}, {0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}, 0xc03acd4760, 0xc075c97700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02fd9b860}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fd9b860}, {0x40, 0xc010cadb70}, 0xc010cadab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02fd9b860}, {0x4e9d7a0, 0xc05b815100}, 0xc03acd4760, 0xc08b141b78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02fd9b860}, 0xc050207620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086e53680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086e53680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28515 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc006114620, {0x63c8510, 0xc01e9a8ec0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:310 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc006114620, {0x63c85b8, 0xc01d3c4300}, 0x20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:335 +0xbb github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x63c85b8, 0xc01d3c4300}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 15892 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc0230d2a80}, 0xc00af3df90, {0xc011896480, 0x11}, 0xc01020ca20) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc0230d2a80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801682 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0982ac548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0982ac5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc072b5fbf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc081fd0a80, {0xc0c8a5e900, 0x2b, 0x52}, {0xc03938dab0, 0x2b, 0x70}, 0x1, 0xfb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc072b5fa88, {0xc0c8a5e900, 0x1, 0x70}, {0xc03938dab0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc072b5fa88, {0xc0c8a5e900, 0x2b, 0x52}, {0xc03938dab0, 0x2b, 0x70}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc072b5f1c0, {0xc03938dab0, 0x70, 0x52}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc072b5f1c0, {0xc0c8a5e900, 0x2b, 0xc03938dab0}, {0xc03938dab0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc072b5ef00, {0xc03938dab0, 0x2b, 0x70}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc047a2bb88, {{0xc04e27d620, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05058bd00, {{0xc04e27d620, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04c96a800, {0x63c85b8, 0xc03c2bd050}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03c2bd050}, {0x64a6560, 0xc05058bd00}, {0xc04e27d620, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03c2bd050}, {0x6466ed8, 0xc047a2bb80}, {0xc04e27d620, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04309f470}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0522f9700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020d8b800, {0x63c85b8, 0xc03c2bd050}, 0xc0522f9700, 0xc06c762aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020d8b800, {0x63c85b8, 0xc03c2bd050}, 0xc0522f9700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c2bcfc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c2bcfc0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c2bcfc0}, {0xc00102e4e0}, 0xc0522f9600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c2bcf60}, 0xc0522f9600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02ef02ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28128 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc01e24fb18) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc02c63d320) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc02c63d320, {0x63c8510, 0xc007b0fa40}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x63c85b8, 0xc02c63d230}, {0xc01e24fc98, 0x2, 0x4000000000000000}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:193 +0x91 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001480420, {0x63c85b8, 0xc02c63d230}, 0xc02ca0c3f0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:291 +0x5de github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x63c85b8, 0xc02ca0c510}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:215 +0x339 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810873 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc03c1752f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28514 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc0122d8800, 0x6437410, 0xc00b38b320, {0x7fe42b743500, 0xc02dc18f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:254 +0x845 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:150 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc0082db200, 0xc00b38b320, {0x6437410, 0xc02dc18f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3082 +0xf7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc00df8d250, 0xc00b38b320, {0x6437410, 0xc02dc18f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:216 +0xf6 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x6437410, 0xc007701038, {0x6437410, 0xc02dc18f60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1227 +0x38 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:753 +0xeb created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:750 +0x1cf goroutine 28396 [select]: github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal({{0x63c85b8, 0xc01c227050}, 0xc014d7c8a0, 0xc014d7c960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:694 +0xbf github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:723 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc00167cf00, {0x63c8510, 0xc0203a8940}, {{0xc02006dee0, 0x3, 0x8}, {0xc02006dee8, 0x3, 0x8}}, {0x1742d9fda89672f3, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:519 +0xd84 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc00167cf00, {0x63c8510, 0xc0203a8940}, 0xc014d7c5a0, {{0xc02006dee0, 0x3, 0x8}, {0xc02006dee8, 0x3, 0x8}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:336 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1({0x63c8510, 0xc0203a8940}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:120 +0xbe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 26212 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc015112dc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc006bb80a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}, 0xc020318798, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}, 0x3, 0xc006bcc080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0xc0016bdce0}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}, 0xc020318798, 0xc006bcc080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ad61800}, {0x460ef4, 0xc003502bc8}, 0xc003502b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}, 0xc020318798, 0xc006bcc080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc002e3dd40}, 0xc020318798, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc0198ce9c0}, 0xc013731200, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0198ce9c0}, 0xc013731200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801916 [runnable]: sync.runtime_SemacquireMutex(0xc04e90f198, 0x8f, 0xc04e90f1a8) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc04e90f040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc04e90f068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0d53bc900, {0xffffffffffffff15, 0xc04e90f1c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0d53bc900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0d53bc900, {0xc0255c4700, 0x32, 0x6e}, {0xc08d103f40, 0x32, 0x93}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09325cd98, {0xc0255c4700, 0x32, 0x6e}, {0xc08d103f40, 0x32, 0x93}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09325b6c0, {0xc08d103f40, 0x93, 0x6e}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09325b6c0, {0xc0255c4700, 0x93, 0xc08d103f40}, {0xc08d103f40, 0xc063347a40, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09325b400, {0xc08d103f40, 0x32, 0x93}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0509802a8, {{0xc063347a40, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05058a800, {{0xc01b4b9f80, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04b729400, {0x63c85b8, 0xc073853f80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc073853f80}, {0x64a6560, 0xc05058a800}, {0xc01b4b9f80, 0xc00bc1fa40, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc073853f80}, {0x6466ed8, 0xc050980000}, {0xc01b4b9f80, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0293d9860}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0d2801f00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020d8d000, {0x63c85b8, 0xc073853f80}, 0xc0d2801f00, 0xc074986fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020d8d000, {0x63c85b8, 0xc073853f80}, 0xc0d2801f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073853ec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073853ec0}, {0xc04e9153f8, 0x660089}, 0xc04e915400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073853ec0}, {0xc0015221e0}, 0xc0d2801c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073853e30}, 0xc0d2801c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}, 0xc0581ba9c0, 0xc064fd32f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}, 0x3, 0xc0d377c540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04e915980}}}, {0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}, 0xc0581ba9c0, 0xc0d377c540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073853e30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073853e30}, {0x40, 0xc04e915b70}, 0xc04e915ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073853e30}, {0x4e9d7a0, 0xc0d2801c00}, 0xc0581ba9c0, 0xc064fd32f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073853e30}, 0xc03db18b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07b02db00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07b02db00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802711 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e72bc00, {0x63c85b8, 0xc0786a29c0}, 0xc0b4d7bf00, 0xc046ded680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc0786a29c0}, 0xc0b4d7bf00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024bf2928, 0x0, 0x0}, 0xc064844800, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024bf2928, 0x0, 0x0}, 0xc064844800, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0786a2930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0786a2930}, {0xc095ad5a08, 0x660089}, 0xc095ad5a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0786a2930}, {0xc00102e4e0}, 0xc0b4d7be00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0786a28d0}, 0xc0b4d7be00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc0786a2840}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0786a2840}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219789587 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe79e58, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0684e7980, 0xc06c92d800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0684e7980, {0xc06c92d800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0684e7980, {0xc06c92d800, 0x735, 0xc0598528c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc02ad6f6f0, {0xc06c92d800, 0x7eda79, 0xc06c92d805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0b012e600, {0xc06c92d800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0342fd5c0, {0xc06c92d800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04ab47778, {0x631b900, 0xc0342fd5c0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04ab47500, {0x7fe42ab59310, 0xc0b012e600}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04ab47500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04ab47500, {0xc086d00000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0ce34e3c0, {0xc086d00000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc068924b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc068924b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0689250e0, {0x634aac0, 0xc068924b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc068924a80, 0xc04e7175c0, 0xc03941542b, 0xc03941542d, {0x635bec0, 0xc04e494bd0}, 0xc000d6ae00, 0xc04df79178, 0xc039415429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc068924a80, {0x63c85b8, 0xc0227362a0}, 0xc04e7175b0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc003cfe038, 0x8}}, 0x0, 0xc022736240, 0xc022736270, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0227362a0}, {0x643d810, 0xc0b012e600}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc020b7c240}, {0x643d810, 0xc0b012e600}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc020b7c240}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219013163 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99788, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06044f900, 0xc03eb8a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06044f900, {0xc03eb8a000, 0x2f61, 0x2f61}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06044f900, {0xc03eb8a000, 0x2f61, 0xc066f8b180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc05013b8f8, {0xc03eb8a000, 0x7eda79, 0xc03eb8a005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0626d1e60, {0xc03eb8a000, 0x3, 0x2f61}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0353000a8, {0xc03eb8a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc08a361e78, {0x631b900, 0xc0353000a8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc08a361c00, {0x7fe42ab59310, 0xc0626d1e60}, 0x2f61) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc08a361c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc08a361c00, {0xc06b473000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04361e4b0, {0xc06b473000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc015566b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc015566b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0155670e0, {0x634aac0, 0xc015566b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc015566a80, 0xc0490ff9c0, 0xc06319142b, 0xc06319142d, {0x635bec0, 0xc04da1d0e0}, 0xc000d6ae00, 0xc0434f12d4, 0xc063191429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc015566a80, {0x63c85b8, 0xc0585a4750}, 0xc0490ff9b0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc00a6c0009, 0xb}}, 0x0, 0xc0585a46f0, 0xc0585a4720, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0585a4750}, {0x643d810, 0xc0626d1e60}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0185fea80}, {0x643d810, 0xc0626d1e60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0185fea80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28148 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001480420, {0x63c8510, 0xc007b0fa40}, 0xc02ca0c3f0, 0xc01df286c0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:331 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x63c8510, 0xc007b0fa40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:288 +0x38 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219808826 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc04962e4b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812761 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812109 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc03c40af90}, 0xc0c884d770, 0x1, 0xc033fb30a8, 0x519bbc, 0xc0662d7c80, 0xc023746960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03c40af90}, 0x120, 0xc033fb3230, 0x662d7c20, 0x0, 0x660069, 0xc0662d7c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc03c40af90}, 0xc0662d7c20, {{{{0x0, 0x0}, {0xc063c79440, 0x30}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03c40af90}, 0x6352410, 0x655a2280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc03c40af90}, {0xc0662d7b00, {0x174301107847188e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc03c40af90}, 0xc064d2a6e0, 0xc033fb39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03c40af90}, 0x660089, {0xc0662d7b00, {0x174301107847188e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc03c40af90}, 0xc07afe3900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107847188e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107847188e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c40af00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c40af00}, {0xc033fb53f8, 0x660089}, 0xc033fb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c40af00}, {0xc0015221e0}, 0xc07afe3700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c40ad80}, 0xc07afe3700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}, 0xc024b39de0, 0xc040c54558) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}, 0x3, 0xc062c9c880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc033fb5980}}}, {0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}, 0xc024b39de0, 0xc062c9c880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c40ad80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c40ad80}, {0x40, 0xc033fb5b70}, 0xc033fb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c40ad80}, {0x4e9d7a0, 0xc07afe3700}, 0xc024b39de0, 0xc040c54558) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c40ad80}, 0xc04a42c060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0662d79e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0662d79e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219790694 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99968, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04b6e0680, 0xc08d1bc000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04b6e0680, {0xc08d1bc000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04b6e0680, {0xc08d1bc000, 0x2faf080, 0x3b9aca00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc047997dd0, {0xc08d1bc000, 0x2faf080, 0x3b9aca00}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0357c4f60, {0xc08d1bc000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc00b73e378, {0xc08d1bc000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc053989e78, {0x631b900, 0xc00b73e378}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc053989c00, {0x7fe42ab59310, 0xc0357c4f60}, 0x4000000000000000) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc053989c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc053989c00, {0xc0b8ce5000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01499cf90, {0xc0b8ce5000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0b279ab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0b279ab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0b279b0e0, {0x634aac0, 0xc0b279ab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0b279aa80, 0xc0544434a0, 0xc09087b42b, 0xc09087b42d, {0x635bec0, 0xc04d603170}, 0xc000d6ae00, 0xc04a426a48, 0xc09087b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0b279aa80, {0x63c85b8, 0xc05a1fa840}, 0xc054443490, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08f83304d, 0x8}}, 0x0, 0xc05a1fa510, 0xc05a1fa810, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc05a1fa840}, {0x643d810, 0xc0357c4f60}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02012f260}, {0x643d810, 0xc0357c4f60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02012f260}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 17331 [runnable]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00c24ce10, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0190c71a0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 219808508 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020d48280, {0x63c85b8, 0xc0571c15c0}, 0xc060ee9470, 0x1, 0x708017fa9e482b, 0x37e11d600, 0xc07af68ba0, 0xc07a148600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0571c15c0}, 0x120, 0xc00e919230, 0x7af68b40, 0x0, 0x660069, 0xc07af68ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020d48280, {0x63c85b8, 0xc0571c15c0}, 0xc07af68b40, {{{{0x0, 0x0}, {0xc033690120, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0571c15c0}, 0x6352410, 0x13b81868) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00a63def0, {0x63c85b8, 0xc0571c15c0}, {0xc07af68a20, {0x174301105b77d610, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00a63def0, {0x63c85b8, 0xc0571c15c0}, 0xc06c51f360, 0xc00e9199f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0571c15c0}, 0x660089, {0xc07af68a20, {0x174301105b77d610, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001920400, {0x63c85b8, 0xc0571c15c0}, 0xc05bc3f400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0571c1500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0571c1500}, {0xc00e91b3f8, 0x660089}, 0xc00e91b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0571c1500}, {0xc0a1fd9ccdcb9c08}, 0xc05bc3f200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0571c1410}, 0xc05bc3f200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}, 0xc047647460, 0xc07e117548) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}, 0x3, 0xc01a45c8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e91b980}}}, {0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}, 0xc047647460, 0xc01a45c8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0571c1410}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0571c1410}, {0x40, 0xc00e91bb70}, 0xc00e91bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0571c1410}, {0x4e9d7a0, 0xc05bc3f200}, 0xc047647460, 0xc07e117548) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0571c1410}, 0xc038a63bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07af68900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07af68900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812756 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804532 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa3c0, {0x63c85b8, 0xc06d424ed0}, 0xc049bf0198, 0x1, 0xc075be90a8, 0x519bbc, 0xc05ef49c80, 0xc058335140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06d424ed0}, 0x120, 0xc075be9230, 0x5ef49c20, 0x0, 0x660069, 0xc05ef49c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa3c0, {0x63c85b8, 0xc06d424ed0}, 0xc05ef49c20, {{{{0x0, 0x0}, {0xc04869f0e0, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06d424ed0}, 0x6352410, 0x21517310) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701180, {0x63c85b8, 0xc06d424ed0}, {0xc05ef49b00, {0x174301103274ea0c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701180, {0x63c85b8, 0xc06d424ed0}, 0xc04151c320, 0xc075be99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06d424ed0}, 0x660089, {0xc05ef49b00, {0x174301103274ea0c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edca800, {0x63c85b8, 0xc06d424ed0}, 0xc056062c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103274ea0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103274ea0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06d424e40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d424e40}, {0xc075beb3f8, 0x660089}, 0xc075beb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06d424e40}, {0x7fe422bcbcb8}, 0xc056062a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06d424d50}, 0xc056062a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}, 0xc045b74180, 0xc031c74900) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}, 0x3, 0xc032aa3640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075beb980}}}, {0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}, 0xc045b74180, 0xc032aa3640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06d424d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d424d50}, {0x40, 0xc075bebb70}, 0xc075bebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06d424d50}, {0x4e9d7a0, 0xc056062a00}, 0xc045b74180, 0xc031c74900) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06d424d50}, 0xc021640900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc058d9ad80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc058d9ad80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799214 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0948f4548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0948f45b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc091e800f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0830cd500, {0xc011a4f680, 0x2b, 0x74}, {0xc097db1c70, 0x2b, 0x4a}, 0x1, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091e7ff88, {0xc011a4f680, 0x1, 0x4a}, {0xc097db1c70, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091e7ff88, {0xc011a4f680, 0x2b, 0x74}, {0xc097db1c70, 0x2b, 0x4a}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091e7f6c0, {0xc097db1c70, 0x4a, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091e7f6c0, {0xc011a4f680, 0x2b, 0xc097db1c70}, {0xc097db1c70, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091e7f400, {0xc097db1c70, 0x2b, 0x4a}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04eda9608, {{0xc04a5e0660, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0a0753a00, {{0xc04a5e0660, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc061b2e800, {0x63c85b8, 0xc075846930}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc075846930}, {0x64a6560, 0xc0a0753a00}, {0xc04a5e0660, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc075846930}, {0x6466ed8, 0xc04eda9600}, {0xc04a5e0660, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc030133c08}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0410a2500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0207d7000, {0x63c85b8, 0xc075846930}, 0xc0410a2500, 0xc04e0b6f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0207d7000, {0x63c85b8, 0xc075846930}, 0xc0410a2500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0758468a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0758468a0}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0758468a0}, {0xc00102e4e0}, 0xc0410a2400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc075846840}, 0xc0410a2400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc78d20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808784 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02fdb6e70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811808 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc0592d5890}, 0xc04170ffb0, 0x1, 0xc0355970a8, 0x519bbc, 0xc04565c960, 0xc010a39b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0592d5890}, 0x120, 0xc035597230, 0x4565c900, 0x0, 0x660069, 0xc04565c960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc0592d5890}, 0xc04565c900, {{{{0x0, 0x0}, {0xc059c9cfc0, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0592d5890}, 0x6352410, 0x65f8a080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc0592d5890}, {0xc04565c7e0, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc0592d5890}, 0xc02d3483c0, 0xc0355979f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0592d5890}, 0x660089, {0xc04565c7e0, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc0592d5890}, 0xc045d30a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592d5800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592d5800}, {0xc0355993f8, 0x660089}, 0xc035599400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592d5800}, {0x7fe42869cda8}, 0xc045d30800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592d5770}, 0xc045d30800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}, 0xc00a67bbc0, 0xc01dc61d28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}, 0x3, 0xc062041f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc035599980}}}, {0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}, 0xc00a67bbc0, 0xc062041f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0592d5770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592d5770}, {0x40, 0xc035599b70}, 0xc035599ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0592d5770}, {0x4e9d7a0, 0xc045d30800}, 0xc00a67bbc0, 0xc01dc61d28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0592d5770}, 0xc023a4ad20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04565c6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04565c6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812770 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 29521 [chan receive]: github.com/cockroachdb/cmux.muxListener.Accept(...) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc01e650b48, {0x63c85b8, 0xc02b3c3f80}, 0xc000d3ab40, {0x6367a10, 0xc00ce5a1e0}, 0xc02e776dc8) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:138 +0x83 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1({0x63c85b8, 0xc02b3c3f80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1369 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810951 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc030af4dc0, {0x63c85b8, 0xc07c4e9ad0}, 0xc06e703620, 0x1, 0xc0065130a8, 0x519bbc, 0xc00d2f69c0, 0xc05938c1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc019473d40, {0x63c85b8, 0xc07c4e9ad0}, 0xc00d2f69c0, 0xc0065131a8, 0x31, 0x40, 0x0, 0xc00d2f69c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc030af4dc0, {0x63c85b8, 0xc07c4e9ad0}, 0xc00d2f6900, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0af323680, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07c4e9ad0}, 0x6352260, 0x51b89790) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0308a02d0, {0x63c85b8, 0xc07c4e9ad0}, {0xc00d2f66c0, {0x1743011035513431, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0308a02d0, {0x63c85b8, 0xc07c4e9ad0}, 0xc059ef72c0, 0xc0065139f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07c4e9ad0}, 0x660089, {0xc00d2f66c0, {0x1743011035513431, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030b12000, {0x63c85b8, 0xc07c4e9ad0}, 0xc059270100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011035513431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011035513431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c4e9a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c4e9a40}, {0xc0065153f8, 0x660089}, 0xc006515400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c4e9a40}, {0x7fe34baaa088}, 0xc08c43bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c4e99b0}, 0xc08c43bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}, 0xc0682d4bc0, 0xc0408ba1c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}, 0x3, 0xc07c0cf780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006515980}}}, {0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}, 0xc0682d4bc0, 0xc07c0cf780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c4e99b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c4e99b0}, {0x40, 0xc006515b70}, 0xc006515ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c4e99b0}, {0x4e9d7a0, 0xc08c43bf00}, 0xc0682d4bc0, 0xc0408ba1c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c4e99b0}, 0xc02c7d79e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00d2f65a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00d2f65a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28484 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc0160ac000, 0x6437410, 0xc00cacb320, {0x7fe42b743500, 0xc00a4128a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:254 +0x845 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:150 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc0082db200, 0xc00cacb320, {0x6437410, 0xc00a4128a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3082 +0xf7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc00a700a30, 0xc00cacb320, {0x6437410, 0xc00a4128a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:216 +0xf6 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x6437410, 0xc012461d88, {0x6437410, 0xc00a4128a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1227 +0x38 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:753 +0xeb created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:750 +0x1cf goroutine 219802607 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc0744b8150}, 0xc040e31900, 0xc06b94b860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0744b8150}, 0xc040e31900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024aa1430, 0x0, 0x0}, 0xc02f0ed470, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024aa1430, 0x0, 0x0}, 0xc02f0ed470, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0744b80c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0744b80c0}, {0xc0588f9a08, 0x660089}, 0xc0588f9a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0744b80c0}, {0xc00102e4e0}, 0xc040e31800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0744b8060}, 0xc040e31800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc04b90c780}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04b90c780}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805442 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc04f1a7170}, 0xc015218ee8, 0x1, 0xc0979d10a8, 0x519bbc, 0xc05b3cd140, 0xc05a8ea2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04f1a7170}, 0x120, 0xc0979d1230, 0x5b3cd0e0, 0xc002400680, 0x18, 0xc05b3cd140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc04f1a7170}, 0xc05b3cd0e0, {{{{0x0, 0x0}, {0xc058cff8c0, 0x24}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f1a7170}, 0x6352410, 0x24ff0f90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc04f1a7170}, {0xc05b3ccfc0, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc04f1a7170}, 0xc065a815e0, 0xc0979d19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f1a7170}, 0x660089, {0xc05b3ccfc0, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc04f1a7170}, 0xc052c3dd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f1a70e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f1a70e0}, {0xc0979d33f8, 0x660089}, 0xc0979d3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f1a70e0}, {0xde83080a2a880100}, 0xc052c3db00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f1a7050}, 0xc052c3db00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}, 0xc0656ebdc0, 0xc015219080) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}, 0x3, 0xc06c770b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0979d3980}}}, {0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}, 0xc0656ebdc0, 0xc06c770b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f1a7050}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f1a7050}, {0x40, 0xc0979d3b70}, 0xc0979d3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f1a7050}, {0x4e9d7a0, 0xc052c3db00}, 0xc0656ebdc0, 0xc015219080) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f1a7050}, 0xc0ce586d80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b3ccea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b3ccea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812390 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0185643c0, {0x63c85b8, 0xc07a157bf0}, 0xc07a0d5e18, 0x1, 0xc0122010a8, 0x519bbc, 0xc0d0fe0300, 0xc05208fbc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0afc30000, {0x63c85b8, 0xc07a157bf0}, 0xc0afc30040, 0xc0122011a8, 0x30, 0x40, 0xffffffffffffffff, 0xc0d0fe0300, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0185643c0, {0x63c85b8, 0xc07a157bf0}, 0xc0d0fe0000, {{{{0x0, 0x0}, {0xc091a5a900, 0xa}}, {{0xc0667e99e0, 0x8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07a157bf0}, 0x63521a0, 0x65f8b120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0185322d0, {0x63c85b8, 0xc07a157bf0}, {0xc013054000, {0x1743011019f90c19, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0185322d0, {0x63c85b8, 0xc07a157bf0}, 0xc02b115040, 0xc0122019f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a157bf0}, 0x660089, {0xc013054000, {0x1743011019f90c19, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018570c00, {0x63c85b8, 0xc07a157bf0}, 0xc057305d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019f90c19, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019f90c19, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a157b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a157b60}, {0xc0122033f8, 0x660089}, 0xc012203400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a157b60}, {0x4086e661e4581d6f}, 0xc057305900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a157a40}, 0xc057305900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}, 0xc061202500, 0xc0731132f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}, 0x3, 0xc0666a4640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc012203980}}}, {0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}, 0xc061202500, 0xc0666a4640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a157a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a157a40}, {0x40, 0xc012203b70}, 0xc012203ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a157a40}, {0x4e9d7a0, 0xc057305900}, 0xc061202500, 0xc0731132f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a157a40}, 0xc050f79ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc058069e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc058069e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28124 [select, 116 minutes]: github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).Start.func1({0x63c85b8, 0xc02ca0c0c0}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:80 +0x31f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807343 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).ResolveIndeterminateCommit(0xc00d502dc0, {0x63c85b8, 0xc03af8d650}, 0xc04087c240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:100 +0x2f7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleIndeterminateCommitError(0xc021a30280, {0x63c85b8, 0xc03af8d650}, 0xc0986c4800, 0xc011e9d260, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:743 +0x4f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021a38400, {0x63c85b8, 0xc03af8d650}, 0xc0986c4800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:503 +0x8d3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffcf1da43, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffcf1da43, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03af8d5c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03af8d5c0}, {0xc0030193f8, 0x660089}, 0xc003019400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03af8d5c0}, {0x7fe323a245d0}, 0xc0986c4600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03af8d500}, 0xc0986c4600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}, 0xc0455de8a0, 0xc0af020780) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}, 0x3, 0xc062e590c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003019980}}}, {0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}, 0xc0455de8a0, 0xc062e590c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03af8d500}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03af8d500}, {0x40, 0xc003019b70}, 0xc003019ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03af8d500}, {0x4e9d7a0, 0xc0986c4600}, 0xc0455de8a0, 0xc0af020780) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03af8d500}, 0xc02fbaf440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc089ef9200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc089ef9200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800145 [semacquire]: sync.runtime_SemacquireMutex(0x43f98e5, 0x0, 0x7fdbe731a880) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0x11869e5, 0x7fe03325e640, 0x7f8a) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc095c6c4f8, 0x11a58d7, 0xc02551f100, 0x7fe03325e601) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7236e27, 0x3b81}, 0x0, 0xc089758ff8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc089758d80, {0xffffffffffffff15, 0xc095c6c5b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc089758d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc089758d80, {0xc0252d4180, 0x32, 0x79}, 0x13, 0x15) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc089758d80, {0xc02a674c80, 0x32, 0x79}, {0xc0252d4180, 0x51c9860, 0xc0252d4180}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc089758d80, {0xc02a674c80, 0x32, 0x73}, {0xc0252d4180, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07b7c0d98, {0xc02a674c80, 0x32, 0x73}, {0xc0252d4180, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07b7bf6c0, {0xc0252d4180, 0x79, 0x73}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07b7bf6c0, {0xc02a674c80, 0x79, 0xc0252d4180}, {0xc0252d4180, 0xc00ab4dea0, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07b7bf400, {0xc0252d4180, 0x32, 0x79}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0480ff428, {{0xc00ab4dea0, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc076190e00, {{0xc032f8ccc0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04fee0400, {0x63c85b8, 0xc074bf9a40}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc074bf9a40}, {0x64a6560, 0xc076190e00}, {0xc032f8ccc0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc074bf9a40}, {0x6466ed8, 0xc0480ff180}, {0xc032f8ccc0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01b5efb00}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc051739d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a35400, {0x63c85b8, 0xc074bf9a40}, 0xc051739d00, 0xc01c7534a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a35400, {0x63c85b8, 0xc074bf9a40}, 0xc051739d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074bf99b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074bf99b0}, {0xc0076ae8f0, 0x660089}, 0xc0076ae8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074bf99b0}, {0xc00102e4e0}, 0xc051739b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074bf9950}, 0xc051739b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b350}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810383 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc05b5356e0}, 0xc049db62d0, 0x1, 0xc02b6390a8, 0x519bbc, 0xc05717fa40, 0xc05f848300) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06c07f4a0, {0x63c85b8, 0xc05b5356e0}, 0xc081bda480, 0xc02b6391a8, 0x5717faa0, 0x0, 0x2, 0xc05717fa40, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc05b5356e0}, 0xc05717f9e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc045559d40, 0xa8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc05b5356e0}, 0x63523b0, 0x51b89120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc05b5356e0}, {0x0, {0x174301106e144fb2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc05b5356e0}, 0xc02dc27cc0, 0xc02b6399f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05b5356e0}, 0x660089, {0x0, {0x174301106e144fb2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc05b5356e0}, 0xc08d4ea900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106e144fb2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106e144fb2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301106e144fb2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05b535650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05b535650}, {0xc02b63b3f8, 0x660089}, 0xc02b63b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05b535650}, {0x0}, 0xc08d4ea700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05b5355c0}, 0xc08d4ea700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}, 0xc07d80d320, 0xc023db3770) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}, 0x3, 0xc027143040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02b63b980}}}, {0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}, 0xc07d80d320, 0xc027143040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05b5355c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05b5355c0}, {0x40, 0xc02b63bb70}, 0xc02b63bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05b5355c0}, {0x4e9d7a0, 0xc08d4ea700}, 0xc07d80d320, 0xc023db3770) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05b5355c0}, 0xc0bfa87e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05717f8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05717f8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805292 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa8c0, {0x63c85b8, 0xc06ecb1bc0}, 0xc05b94fea8, 0x1, 0xc06781b0a8, 0x519bbc, 0xc0590bd8c0, 0xc0577ceba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06ecb1bc0}, 0x51cdcd8, 0xc06781b230, 0x6781b138, 0x4c166a, 0x18, 0xc0590bd8c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa8c0, {0x63c85b8, 0xc06ecb1bc0}, 0xc0590bd860, {{{{0x0, 0x0}, {0xc051d79e60, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc02c1af740, {0x63c85b8, 0xc06ecb1bc0}, 0xc06b278090, 0x6781b428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701400, {0x63c85b8, 0xc06ecb1bc0}, {0xc05b3cc360, {0x174301103edcf633, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701400, {0x63c85b8, 0xc06ecb1bc0}, 0xc05f814f00, 0xc06781b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06ecb1bc0}, 0x660089, {0xc05b3cc360, {0x174301103edcf633, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ede5800, {0x63c85b8, 0xc06ecb1bc0}, 0xc052c3d500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103edcf633, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103edcf633, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ecb1b30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ecb1b30}, {0xc06781d3f8, 0x660089}, 0xc06781d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ecb1b30}, {0x7fe3b8ab22d0}, 0xc052c3d300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06ecb1aa0}, 0xc052c3d300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}, 0xc0656ebbe0, 0xc015218c90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}, 0x3, 0xc027999f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06781d980}}}, {0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}, 0xc0656ebbe0, 0xc027999f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06ecb1aa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ecb1aa0}, {0x40, 0xc06781db70}, 0xc06781dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}, 0x203009, 0x203009) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06ecb1aa0}, {0x4e9d7a0, 0xc052c3d300}, 0xc0656ebbe0, 0xc015218c90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06ecb1aa0}, 0xc0ce586780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ace3680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ace3680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811305 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021c5a280, {0x63c85b8, 0xc08d943500}, 0xc0a7bf7740, 0x1, 0xc0293370a8, 0x519bbc, 0xc0d05e2940, 0xc0783be0c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0c4745b00, {0x63c85b8, 0xc08d943500}, 0xc0c4745b38, 0xc0293371a8, 0x30, 0x40, 0xffffffffffffffff, 0xc0d05e2940, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021c5a280, {0x63c85b8, 0xc08d943500}, 0xc0d05e2700, {{{{0xc01e203440, 0x2}, {0xc04869fe60, 0x2}}, {{0xc0d6108000, 0x7}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08d943500}, 0x63521a0, 0x3c750438) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c42460, {0x63c85b8, 0xc08d943500}, {0xc0624e3c20, {0x174301100e2410da, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c42460, {0x63c85b8, 0xc08d943500}, 0xc079424000, 0xc0293379f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08d943500}, 0x660089, {0xc0624e3c20, {0x174301100e2410da, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c68c00, {0x63c85b8, 0xc08d943500}, 0xc04eed6700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100e2410da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100e2410da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08d943440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d943440}, {0xc0293393f8, 0x660089}, 0xc029339400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08d943440}, {0xc0015221e0}, 0xc04eed6400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08d943320}, 0xc04eed6400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}, 0xc028541ea0, 0xc01c104a80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}, 0x3, 0xc0c8d827c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc029339980}}}, {0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}, 0xc028541ea0, 0xc0c8d827c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08d943320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d943320}, {0x40, 0xc029339b70}, 0xc029339ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08d943320}, {0x4e9d7a0, 0xc04eed6400}, 0xc028541ea0, 0xc01c104a80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08d943320}, 0xc04abe2a20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05073c480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05073c480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806390 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020f88500, {0x63c85b8, 0xc07a12b350}, 0xc089f8dcc8, 0x1, 0xc004e4f0a8, 0x519bbc, 0xc026f9e240, 0xc01bcdb8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07a12b350}, 0x1e0, 0xc004e4f230, 0x26f9e1e0, 0x0, 0x660069, 0xc026f9e240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020f88500, {0x63c85b8, 0xc07a12b350}, 0xc026f9e1e0, {{{{0x0, 0x0}, {0xc042834360, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07a12b350}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc020df9ef0, {0x63c85b8, 0xc07a12b350}, {0xc046333560, {0x174301104c829237, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc020df9ef0, {0x63c85b8, 0xc07a12b350}, 0xc083c9e8c0, 0xc004e4f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a12b350}, 0x660089, {0xc046333560, {0x174301104c829237, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210be000, {0x63c85b8, 0xc07a12b350}, 0xc0674b8500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c829237, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c829237, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a12b2c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a12b2c0}, {0xc004e513f8, 0x660089}, 0xc004e51400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a12b2c0}, {0x7fe4287c3510}, 0xc0674b8300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a12b230}, 0xc0674b8300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}, 0xc0823b5b20, 0xc08f5647c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}, 0x3, 0xc0754ddb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004e51980}}}, {0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}, 0xc0823b5b20, 0xc0754ddb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a12b230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a12b230}, {0x40, 0xc004e51b70}, 0xc004e51ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a12b230}, {0x4e9d7a0, 0xc0674b8300}, 0xc0823b5b20, 0xc08f5647c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a12b230}, 0xc0629c3aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc046333440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc046333440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794447 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc01191db60}, 0xc032b1bcb0, {0x646cc48, 0xc01191dc80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc01191db60}, 0xedb7a834e, 0xc0501d7e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc01191db60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0724928d0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc032b1bef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0724928d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799212 [semacquire]: sync.runtime_SemacquireMutex(0xc002401a00, 0x3, 0xc002401a00) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc08ebd0338) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fe1c5d10280, 0x7fb7, 0x7fd46018dc40) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc021c1e1c0, 0x7fe1c5d10201, 0x7fe1c5d102a0, 0x42ca, 0x42cf) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc022192580, {0x141adf8, 0x42ca}, 0x0, 0xc06c457478) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc06c457200, {0xffffffffffffff15, 0xc08ebd05c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc06c457200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc06c457200, {0xc08f7bac60, 0x2b, 0x8c}, 0x13, 0xf3) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc06c457200, {0xc082f93b90, 0x2b, 0x8c}, {0xc08f7bac60, 0x51c9860, 0xc08f7bac60}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc06c457200, {0xc082f93b90, 0x2b, 0x90}, {0xc08f7bac60, 0x2b, 0x8c}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092850d98, {0xc082f93b90, 0x2b, 0x90}, {0xc08f7bac60, 0x2b, 0x8c}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09284f6c0, {0xc08f7bac60, 0x8c, 0x90}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09284f6c0, {0xc082f93b90, 0x2b, 0xc08f7bac60}, {0xc08f7bac60, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09284f400, {0xc08f7bac60, 0x2b, 0x8c}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc03dece108, {{0xc07099e330, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc076085d00, {{0xc07099e330, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0523c8c00, {0x63c85b8, 0xc079813ef0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc079813ef0}, {0x64a6560, 0xc076085d00}, {0xc07099e330, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc079813ef0}, {0x6466ed8, 0xc03dece100}, {0xc07099e330, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07c51bdd0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0339b0700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0207d5800, {0x63c85b8, 0xc079813ef0}, 0xc0339b0700, 0xc016145c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0207d5800, {0x63c85b8, 0xc079813ef0}, 0xc0339b0700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xc, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xc, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xc, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079813e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079813e60}, {0xc0032fc8f0, 0x660089}, 0xc0032fc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079813e60}, {0xc00102e4e0}, 0xc0339b0600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079813e00}, 0xc0339b0600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xc, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc78c60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806635 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018383400, {0x63c85b8, 0xc08f56f7a0}, 0xc09ca42870, 0x1, 0xc0031d76d8, 0x519bbc, 0xc048f7b1a0, 0xc0494bbce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc07c20a000, {0x63c85b8, 0xc08f56f7a0}, 0xc048f7b1a0, 0xc0031d77d8, 0x30, 0x40, 0x0, 0xc048f7b1a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018383400, {0x63c85b8, 0xc08f56f7a0}, 0xc048f7b0e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc07ca00c60, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08f56f7a0}, 0x6352260, 0x53414838) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0181dfb30, {0x63c85b8, 0xc08f56f7a0}, {0xc048f7afc0, {0x174301101615db41, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0181dfb30, {0x63c85b8, 0xc08f56f7a0}, 0xc0189c4280, 0xc0031d8020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08f56f7a0}, 0x660089, {0xc048f7afc0, {0x174301101615db41, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0184f7c00, {0x63c85b8, 0xc08f56f7a0}, 0xc097d96c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101615db41, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101615db41, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08f56f710}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08f56f710}, {0xc089d55a28, 0x660089}, 0xc089d55a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08f56f710}, {0xc00102e4e0}, 0xc097d96b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08f56f6b0}, 0xc097d96b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc046863350, {0x63c85b8, 0xc08f56f620}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc046863350, {0x63c85b8, 0xc08f56f620}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc08f56f620}, {0x4f67ad2, 0x6446ff8}, 0xc01ad23fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219810161 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0305d3000, {0x63c85b8, 0xc0662857a0}, 0xc041cc5100, 0xc0861d0960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0305d3000, {0x63c85b8, 0xc0662857a0}, 0xc041cc5100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c367d6e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c367d6e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066285710}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066285710}, {0xc01fb1b3f8, 0x660089}, 0xc01fb1b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066285710}, {0xc0015221e0}, 0xc041cc4f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066285680}, 0xc041cc4f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}, 0xc06fc90ec0, 0xc050c60870) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}, 0x3, 0xc06cc45100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fb1b980}}}, {0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}, 0xc06fc90ec0, 0xc06cc45100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066285680}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066285680}, {0x40, 0xc01fb1bb70}, 0xc01fb1bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066285680}, {0x4e9d7a0, 0xc041cc4f00}, 0xc06fc90ec0, 0xc050c60870) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066285680}, 0xc0c8ac03c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0830d0c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0830d0c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28499 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc0122d8800, 0x6437410, 0xc01978efc0, {0x7fe42b743500, 0xc00863eea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:254 +0x845 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:150 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc0082db200, 0xc01978efc0, {0x6437410, 0xc00863eea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3082 +0xf7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc00df8d250, 0xc01978efc0, {0x6437410, 0xc00863eea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:216 +0xf6 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x6437410, 0xc012461b00, {0x6437410, 0xc00863eea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1227 +0x38 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:753 +0xeb created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:750 +0x1cf goroutine 219790731 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc03a7438f0}, 0xc0ae80d800, 0xc062a2a0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03a7438f0}, 0xc0ae80d800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03a743800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a743800}, {0xc09b26c7f8, 0x660089}, 0xc09b26c800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03a743800}, {0xc00102e4e0}, 0xc0ae80d700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a7437a0}, 0xc0ae80d700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc066ea80b0, {0x63c85b8, 0xc03a743650}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc06b910600, {0x63c85b8, 0xc03a743650}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc06b910600, {0x63c8510, 0xc00ecc7f40}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc06b910600, {0x63c8510, 0xc00ecc7f40}, {0x63f7db8, 0x9bf6f48}, 0xc09b271088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc06b910600, {0x63c85b8, 0xc0828a2840}, {{0x63f7db8, 0x9bf6f48}, {0xc06e80f27c, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc06b910600, {0x63c85b8, 0xc0828a2840}, {{0x63f7db8, 0x9bf6f48}, {0xc06e80f27c, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc06e80f27c, 0x6}, 0x0, 0x0}, {0x1eb5df37, 0xedb7a834f, 0x0}, {0x1eb5e9a7, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06b910600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06b910600, {0x63c8510, 0xc0c3cb9540}, 0xc076c8bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0544434c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0c3cb9540}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0544434c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812560 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc04f5ed8c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0612d4000, 0xc042303168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04f5ed8c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0612d4000, {0xc0cabb2180, 0x29, 0x72}, {0xc0336b2d20, 0x29, 0x6e}, 0x0, 0xfb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092d7fa38, {0xc0cabb2180, 0x1, 0x6e}, {0xc0336b2d20, 0xc092d7ea00, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092d7fa38, {0xc0cabb2180, 0x29, 0x72}, {0xc0336b2d20, 0x29, 0x6e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092d7ecc0, {0xc0336b2d20, 0x0, 0xc0423034e0}, 0x121ea30, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092d7ecc0, {0xc0cabb2180, 0x29, 0x6e}, {0xc0336b2d20, 0x28, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092d7ea00, {0xc0336b2d20, 0x29, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc051349188, {{0xc03a992e10, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc046381f00, {{0xc03a992e10, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07347d400, {0x63c85b8, 0xc0566156e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0566156e0}, {0x64a6560, 0xc046381f00}, {0xc03a992e10, 0x0, 0x30}, {0x174301109513eb60, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0566156e0}, {0x6466ed8, 0xc051349180}, {0xc03a992e10, 0x28, 0x30}, {0x174301109513eb60, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc064a06048}, {{0x174301109513eb60, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109513eb60, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046448d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b2e15060, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0312b0400, {0x63c85b8, 0xc0566156e0}, 0xc046448d00, 0xc01e1e81e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0312b0400, {0x63c85b8, 0xc0566156e0}, 0xc046448d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109513eb60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109513eb60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056615650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056615650}, {0xc0423093f8, 0x660089}, 0xc042309400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056615650}, {0xc0015221e0}, 0xc046448b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0566155c0}, 0xc046448b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}, 0xc0ccf7ff20, 0xc038161fe0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}, 0x3, 0xc071e55640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042309980}}}, {0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}, 0xc0ccf7ff20, 0xc071e55640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0566155c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0566155c0}, {0x40, 0xc042309b70}, 0xc042309ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0566155c0}, {0x4e9d7a0, 0xc046448b00}, 0xc0ccf7ff20, 0xc038161fe0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0566155c0}, 0xc071983a40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea3320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea3320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 15949 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc0160acc00, 0x6437410, 0xc0135d7320, {0x7fe42b743500, 0xc01cd30da0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:254 +0x845 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:150 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc0082db200, 0xc0135d7320, {0x6437410, 0xc01cd30da0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3082 +0xf7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc00185abd0, 0xc0135d7320, {0x6437410, 0xc01cd30da0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:216 +0xf6 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x6437410, 0xc018b008b8, {0x6437410, 0xc01cd30da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1227 +0x38 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:753 +0xeb created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:750 +0x1cf goroutine 219782146 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a599a0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc064107b00, 0xc053d95000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc064107b00, {0xc053d95000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc064107b00, {0xc053d95000, 0x735, 0xc0526d5a40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc04831ddd0, {0xc053d95000, 0x7eda79, 0xc053d95005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0c647d020, {0xc053d95000, 0xc09c10ac60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04a6d9998, {0xc053d95000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0539d5af8, {0x631b900, 0xc04a6d9998}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0539d5880, {0x7fe42ab59310, 0xc0c647d020}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0539d5880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0539d5880, {0xc0af072000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02ddbfe60, {0xc0af072000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc001082b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc001082b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0010830e0, {0x634aac0, 0xc001082b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc001082a80, 0xc04a843cd0, 0xc09c10b42b, 0xc09c10b42d, {0x635bec0, 0xc012661a70}, 0xc000d6ae00, 0xc03f0ce898, 0xc09c10b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc001082a80, {0x63c85b8, 0xc076ed5b90}, 0xc04a843cc0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc067500038, 0x8}}, 0x0, 0xc076ed5b30, 0xc076ed5b60, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc076ed5b90}, {0x643d810, 0xc0c647d020}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc052500450}, {0x643d810, 0xc0c647d020}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc052500450}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811912 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc078028b10}, 0xc051b45068, 0x1, 0xc033c71ff0, 0x519bbc, 0xc0516d1500, 0xc0401f6060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc078028b10}, 0x1e0, 0xc033c72178, 0x516d14a0, 0x0, 0x660069, 0xc0516d1500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc078028b10}, 0xc0516d14a0, {{{{0x0, 0x0}, {0xc0af5be480, 0x70}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc078028b10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc078028b10}, {0xc00e987c20, {0x174301107c3de5b3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc078028b10}, 0xc06191a960, 0xc06f204938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc078028b10}, 0x660089, {0xc00e987c20, {0x174301107c3de5b3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc078028b10}, 0xc01f6fe500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107c3de5b3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107c3de5b3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078028a80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078028a80}, {0xc06f206340, 0x660089}, 0xc06f206348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078028a80}, {0xc00102e4e0}, 0xc01f6fe300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078028a20}, 0xc01f6fe300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc078028570}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808228 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06c800, {0x63c85b8, 0xc081d0eff0}, 0xc04ed2f700, 0xc02c9ff220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc081d0eff0}, 0xc04ed2f700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a7a2ce9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a7a2ce9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc081d0ef60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc081d0ef60}, {0xc005eb53f8, 0x660089}, 0xc005eb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc081d0ef60}, {0x4044fffcb3216cf9}, 0xc04ed2f500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc081d0eea0}, 0xc04ed2f500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}, 0xc09d3b77c0, 0xc01c026948) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}, 0x3, 0xc064cc0340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005eb5980}}}, {0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}, 0xc09d3b77c0, 0xc064cc0340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc081d0eea0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc081d0eea0}, {0x40, 0xc005eb5b70}, 0xc005eb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc081d0eea0}, {0x4e9d7a0, 0xc04ed2f500}, 0xc09d3b77c0, 0xc01c026948) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc081d0eea0}, 0xc075b390e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc072f77680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc072f77680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812355 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc031a1c990}, 0xc054533728, 0x1, 0xc01fbd9ff0, 0x519bbc, 0xc059ec9c80, 0xc027eb78c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc031a1c990}, 0x1e0, 0xc01fbda178, 0x59ec9c20, 0x0, 0x660069, 0xc059ec9c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc031a1c990}, 0xc059ec9c20, {{{{0x0, 0x0}, {0xc03081f200, 0x6d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc031a1c990}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc031a1c990}, {0xc05ba3ad80, {0x1743011080a716be, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc031a1c990}, 0xc055e5e280, 0xc07c238938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc031a1c990}, 0x660089, {0xc05ba3ad80, {0x1743011080a716be, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc031a1c990}, 0xc06d230500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011080a716be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011080a716be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031a1c8d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031a1c8d0}, {0xc07c23a340, 0x660089}, 0xc07c23a348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031a1c8d0}, {0xc00102e4e0}, 0xc06d230400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031a1c870}, 0xc06d230400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc046e1a870}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805045 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc02242aa80}, 0xc03c2efa10, 0x1, 0x708017da5c7e59, 0x37e11d600, 0xc07013bc80, 0xc05af415c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02242aa80}, 0x120, 0xc091767230, 0x7013bc20, 0xc00b8b7380, 0x18, 0xc07013bc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc02242aa80}, 0xc07013bc20, {{{{0x0, 0x0}, {0xc09298fd40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02242aa80}, 0x6352410, 0x18060230) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc02242aa80}, {0xc07013bb00, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc02242aa80}, 0xc03babf9a0, 0xc0917679f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02242aa80}, 0x660089, {0xc07013bb00, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc02242aa80}, 0xc0622bb200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02242a8a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02242a8a0}, {0xc0917693f8, 0x660089}, 0xc091769400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02242a8a0}, {0x7fe426af1a10}, 0xc0622baf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02242a270}, 0xc0622baf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}, 0xc0664ffd40, 0xc03c2ef9c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}, 0x3, 0xc060d7ffc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc091769980}}}, {0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}, 0xc0664ffd40, 0xc060d7ffc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02242a270}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02242a270}, {0x40, 0xc091769b70}, 0xc091769ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}, 0x203018, 0x203018) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02242a270}, {0x4e9d7a0, 0xc0622baf00}, 0xc0664ffd40, 0xc03c2ef9c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02242a270}, 0xc0c3a60a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28498 [select]: github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal({{0x63c85b8, 0xc01be13e00}, 0xc01425f800, 0xc01504e000}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:694 +0xbf github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:723 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc00167cf00, {0x63c8510, 0xc017d65400}, {{0xc0200c0930, 0x1, 0x8}, {0xc0200c0938, 0x1, 0x8}}, {0x1742d9fda845eb13, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:519 +0xd84 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc00167cf00, {0x63c8510, 0xc017d65400}, 0xc01425f5c0, {{0xc0200c0930, 0x1, 0x8}, {0xc0200c0938, 0x1, 0x8}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:336 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1({0x63c8510, 0xc017d65400}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:120 +0xbe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219811610 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc03e5960c0}, 0xc000f13608, 0x1, 0xc0358910a8, 0x519bbc, 0xc046a30600, 0xc065bbe420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03e5960c0}, 0x1e0, 0xc035891230, 0x46a305a0, 0x0, 0x660069, 0xc046a30600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc03e5960c0}, 0xc046a305a0, {{{{0x0, 0x0}, {0xc0752778c0, 0x68}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc0911da4e0, {0x63c85b8, 0xc03e5960c0}, 0xc008500cf0, 0x35891428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc03e5960c0}, {0xc063c9ac60, {0x174301107b1b7318, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc03e5960c0}, 0xc065099e00, 0xc0358919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03e5960c0}, 0x660089, {0xc063c9ac60, {0x174301107b1b7318, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc03e5960c0}, 0xc07afa8700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b1b7318, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b1b7318, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03e596030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03e596030}, {0xc0358933f8, 0x660089}, 0xc035893400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03e596030}, {0x7fe34e25da88}, 0xc07afa8500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b915f20}, 0xc07afa8500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}, 0xc044a36c20, 0xc000f13ba8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}, 0x3, 0xc063201e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc035893980}}}, {0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}, 0xc044a36c20, 0xc063201e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b915f20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b915f20}, {0x40, 0xc035893b70}, 0xc035893ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b915f20}, {0x4e9d7a0, 0xc07afa8500}, 0xc044a36c20, 0xc000f13ba8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b915f20}, 0xc020b98c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc063c9ab40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc063c9ab40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807373 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc082fb69f0}, 0xc0984bed00, 0xc0761d03c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc082fb69f0}, 0xc0984bed00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110576a178b, 0x0, 0x0}, 0xc0906908b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110576a178b, 0x0, 0x0}, 0xc0906908b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc082fb6960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082fb6960}, {0xc0040a93f8, 0x660089}, 0xc0040a9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc082fb6960}, {0x0}, 0xc0984bea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc082fb68d0}, 0xc0984bea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}, 0xc07b4d7720, 0xc06605ad38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}, 0x3, 0xc08872bd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0040a9980}}}, {0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}, 0xc07b4d7720, 0xc08872bd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc082fb68d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082fb68d0}, {0x40, 0xc0040a9b70}, 0xc0040a9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc082fb68d0}, {0x4e9d7a0, 0xc0984bea00}, 0xc07b4d7720, 0xc06605ad38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc082fb68d0}, 0xc02420a840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08762a900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08762a900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811186 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024ab5cc0, {0x63c85b8, 0xc03c51b4a0}, 0xc04584e8e8, 0x1, 0xc026fd90a8, 0x519bbc, 0xc03bf5cf00, 0xc07057ccc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03c51b4a0}, 0x120, 0xc026fd9230, 0x3bf5cea0, 0x0, 0x660069, 0xc03bf5cf00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024ab5cc0, {0x63c85b8, 0xc03c51b4a0}, 0xc03bf5cea0, {{{{0x0, 0x0}, {0xc0a2e26360, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03c51b4a0}, 0x6352410, 0x6b245b98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024c045f0, {0x63c85b8, 0xc03c51b4a0}, {0xc03bf5cd80, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024c045f0, {0x63c85b8, 0xc03c51b4a0}, 0xc077d0a500, 0xc026fd99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03c51b4a0}, 0x660089, {0xc03bf5cd80, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024c29000, {0x63c85b8, 0xc03c51b4a0}, 0xc034da3d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c51b260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c51b260}, {0xc026fdb3f8, 0x660089}, 0xc026fdb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c51b260}, {0xc01b5bdae0}, 0xc034da3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c51b140}, 0xc034da3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}, 0xc05a440420, 0xc0668d1398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}, 0x3, 0xc06ad65ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026fdb980}}}, {0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}, 0xc05a440420, 0xc06ad65ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c51b140}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c51b140}, {0x40, 0xc026fdbb70}, 0xc026fdbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c51b140}, {0x4e9d7a0, 0xc034da3b00}, 0xc05a440420, 0xc0668d1398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c51b140}, 0xc06e273380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03bf5cc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03bf5cc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803692 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0122d9400, {0x63c85b8, 0xc04f4c62a0}, 0xc08af95d00, 0xc03c3ee500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0122d9400, {0x63c85b8, 0xc04f4c62a0}, 0xc08af95d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102edfba16, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102edfba16, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f4c6210}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f4c6210}, {0xc076cc33f8, 0x660089}, 0xc076cc3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f4c6210}, {0xc045e0e8a0}, 0xc08af95b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f4c6180}, 0xc08af95b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}, 0xc061a121e0, 0xc0aba0ded8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}, 0x3, 0xc02800a680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076cc3980}}}, {0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}, 0xc061a121e0, 0xc02800a680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f4c6180}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f4c6180}, {0x40, 0xc076cc3b70}, 0xc076cc3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f4c6180}, {0x4e9d7a0, 0xc08af95b00}, 0xc061a121e0, 0xc0aba0ded8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f4c6180}, 0xc01b6a3380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0436a98c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0436a98c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809041 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020d99b80, {0x63c85b8, 0xc0183a0180}, 0xc032aacca8, 0x1, 0xc0248e30a8, 0x519bbc, 0xc034ef5ec0, 0xc06e551920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0183a0180}, 0x120, 0xc0248e3230, 0x34ef5e60, 0x0, 0x660069, 0xc034ef5ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020d99b80, {0x63c85b8, 0xc0183a0180}, 0xc034ef5e60, {{{{0x0, 0x0}, {0xc012142c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0183a0180}, 0x6352410, 0x47996798) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc020df8af0, {0x63c85b8, 0xc0183a0180}, {0xc034ef5d40, {0x174301105b86b204, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc020df8af0, {0x63c85b8, 0xc0183a0180}, 0xc01edfe820, 0xc0248e39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0183a0180}, 0x660089, {0xc034ef5d40, {0x174301105b86b204, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020edc000, {0x63c85b8, 0xc0183a0180}, 0xc058b81e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b86b204, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b86b204, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0183a00f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0183a00f0}, {0xc0248e53f8, 0x660089}, 0xc0248e5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0183a00f0}, {0x3f318bcfb232913d}, 0xc058b81c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0183a0030}, 0xc058b81c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}, 0xc044c78560, 0xc041811290) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}, 0x3, 0xc01cdfe680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0248e5980}}}, {0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}, 0xc044c78560, 0xc01cdfe680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0183a0030}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0183a0030}, {0x40, 0xc0248e5b70}, 0xc0248e5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0183a0030}, {0x4e9d7a0, 0xc058b81c00}, 0xc044c78560, 0xc041811290) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0183a0030}, 0xc02916eba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc034ef5c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc034ef5c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219792514 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc023fe03c0}, 0xc05fb47cb0, {0x646cc48, 0xc023fe0540}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc023fe03c0}, 0xedb7a834e, 0xc06acbea20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc023fe03c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0290a21b0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc05fb47ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0290a21b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 15890 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0036c4680) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 219812559 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc044fe8700, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc0421d1198, 0xc079deba38, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc079debb50, 0x0, 0xc079debb50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc079deb9e8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc079deb9e8, {0xc03419b270, 0x18, 0x4c}, {0xc0b0fa3420, 0x18, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc079dea7c0, {0xc0b0fa3420, 0x0, 0xc0421d14e0}, 0x121ea30, 0x20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc079dea7c0, {0xc03419b270, 0x18, 0x6d}, {0xc0b0fa3420, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc079dea500, {0xc0b0fa3420, 0x18, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc070169088, {{0xc0578ee5e8, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0839f5c00, {{0xc0578ee5e8, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0840dc400, {0x63c85b8, 0xc0566154a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0566154a0}, {0x64a6560, 0xc0839f5c00}, {0xc0578ee5e8, 0x0, 0x18}, {0x17430110950be0cf, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0566154a0}, {0x6466ed8, 0xc070169080}, {0xc0578ee5e8, 0x17, 0x18}, {0x17430110950be0cf, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02201c330}, {{0x17430110950be0cf, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110950be0cf, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046448a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b2d945cf, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc0566154a0}, 0xc046448a00, 0xc04d97bcc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc0566154a0}, 0xc046448a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110950be0cf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110950be0cf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056615380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056615380}, {0xc0421d73f8, 0x660089}, 0xc0421d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056615380}, {0xc0015221e0}, 0xc046448800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0566152f0}, 0xc046448800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}, 0xc0ccf7fec0, 0xc038161f80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}, 0x3, 0xc071e55480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0421d7980}}}, {0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}, 0xc0ccf7fec0, 0xc071e55480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0566152f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0566152f0}, {0x40, 0xc0421d7b70}, 0xc0421d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0566152f0}, {0x4e9d7a0, 0xc046448800}, 0xc0ccf7fec0, 0xc038161f80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0566152f0}, 0xc071983800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea3200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea3200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812567 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28321 [select]: github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal({{0x63c85b8, 0xc01cbb5e00}, 0xc0162728a0, 0xc016272900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:694 +0xbf github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:723 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc00167cf00, {0x63c8510, 0xc02aec1080}, {{0xc01fa95268, 0x1, 0x8}, {0xc01fa95270, 0x1, 0x8}}, {0x1742d9fda86ab627, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:519 +0xd84 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc00167cf00, {0x63c8510, 0xc02aec1080}, 0xc015ca4d80, {{0xc01fa95268, 0x1, 0x8}, {0xc01fa95270, 0x1, 0x8}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:336 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1({0x63c8510, 0xc02aec1080}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:120 +0xbe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219811238 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc01b20ba40}, 0xc053003590, 0x1, 0x0, 0x1, 0xc01a710060, 0xc03de15140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01b20ba40}, 0x120, 0xc02868e728, 0x1a710000, 0x0, 0x660069, 0xc01a710060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc01b20ba40}, 0xc01a710000, {{{{0x0, 0x0}, {0xc04d19e900, 0x106}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01b20ba40}, 0x6352410, 0x412adde8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc01b20ba40}, {0xc01e7667e0, {0x1743011073bcd08b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc01b20ba40}, 0xc0682a6000, 0xc06318aee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01b20ba40}, 0x660089, {0xc01e7667e0, {0x1743011073bcd08b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01b20ba40}, 0xc049b40d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011073bcd08b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011073bcd08b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b20b890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b20b890}, {0xc0ad83c8f0, 0x660089}, 0xc0ad83c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b20b890}, {0xc00102e4e0}, 0xc049b40c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b20b7a0}, 0xc049b40c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc059890e10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811640 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc051a20840}, 0xc0beab8ed0, 0x1, 0xc0340270a8, 0x519bbc, 0xc0364ee960, 0xc052552600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc051a20840}, 0x120, 0xc034027230, 0x364ee900, 0x0, 0x660069, 0xc0364ee960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc051a20840}, 0xc0364ee900, {{{{0x0, 0x0}, {0xc09ff566c0, 0x2f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc051a20840}, 0x6352410, 0x46738120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc051a20840}, {0xc0364ee7e0, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc051a20840}, 0xc051e62f00, 0xc0340279f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc051a20840}, 0x660089, {0xc0364ee7e0, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc051a20840}, 0xc07ad03300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc051a207b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051a207b0}, {0xc0340293f8, 0x660089}, 0xc034029400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc051a207b0}, {0x7fe2ff5b2280}, 0xc07ad03100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc051a20720}, 0xc07ad03100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}, 0xc028223e40, 0xc0433baaf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}, 0x3, 0xc044803380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034029980}}}, {0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}, 0xc028223e40, 0xc044803380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc051a20720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051a20720}, {0x40, 0xc034029b70}, 0xc034029ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc051a20720}, {0x4e9d7a0, 0xc07ad03100}, 0xc028223e40, 0xc0433baaf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc051a20720}, 0xc01b732360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc065ff2fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc065ff2fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806156 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02f7f5710}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804654 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc041b34790, {0x63c85b8, 0xc03062b4d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc08f3c8000, {0x63c85b8, 0xc03062b4d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc08f3c8000, {0x63c8510, 0xc051705a40}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc08f3c8000, {0x63c8510, 0xc051705a40}, 0xc0429d7400, {0x7fe42b790cf0, 0xc053f2c6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc08f3c8000, {0x63c85b8, 0xc08dbd5da0}, {{0x63f88f8, 0xc0429d7400}, {0xc09381a8a0, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc08f3c8000, {0x63c85b8, 0xc08dbd5da0}, {{0x63f88f8, 0xc0429d7400}, {0xc09381a8a0, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0429d7400}, {0xc09381a8a0, 0x23}, 0x0, 0x0}, {0x4536477, 0xedb7a8350, 0x0}, {0x4536d54, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08f3c8000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08f3c8000, {0x63c8510, 0xc03e23eb80}, 0xc057c77dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0cd9401b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03e23eb80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0cd9401b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219803337 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc01fb5d230}, 0xc0b5c14450, 0x1, 0xc052bd30a8, 0x519bbc, 0xc044be5920, 0xc04bd73140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01fb5d230}, 0x120, 0xc052bd3230, 0x44be58c0, 0x0, 0x660069, 0xc044be5920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc01fb5d230}, 0xc044be58c0, {{{{0x0, 0x0}, {0xc03d108900, 0x44}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01fb5d230}, 0x6352410, 0x41bc4da8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc01fb5d230}, {0xc044be57a0, {0x1743011025e86b69, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc01fb5d230}, 0xc041b808c0, 0xc052bd39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01fb5d230}, 0x660089, {0xc044be57a0, {0x1743011025e86b69, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc01fb5d230}, 0xc0c5e1cd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025e86b69, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025e86b69, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fb5d1a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fb5d1a0}, {0xc052bd53f8, 0x660089}, 0xc052bd5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fb5d1a0}, {0xc0320a5820}, 0xc0c5e1cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fb5d110}, 0xc0c5e1cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}, 0xc0276bcf40, 0xc02684d3c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}, 0x3, 0xc065e5b8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc052bd5980}}}, {0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}, 0xc0276bcf40, 0xc065e5b8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01fb5d110}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fb5d110}, {0x40, 0xc052bd5b70}, 0xc052bd5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01fb5d110}, {0x4e9d7a0, 0xc0c5e1cb00}, 0xc0276bcf40, 0xc02684d3c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01fb5d110}, 0xc029def380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc044be46c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc044be46c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809487 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0716d3830}, 0xc0284fe450, 0x1, 0xc00a64d0a8, 0x519bbc, 0xc086e52720, 0xc091614f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0716d3830}, 0x120, 0xc00a64d230, 0x86e526c0, 0x0, 0x660069, 0xc086e52720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0716d3830}, 0xc086e526c0, {{{{0x0, 0x0}, {0xc060ff7d40, 0xd9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0716d3830}, 0x6352410, 0x6bab3db8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0716d3830}, {0xc086e525a0, {0x174301106409cf95, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0716d3830}, 0xc05b686140, 0xc00a64d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0716d3830}, 0x660089, {0xc086e525a0, {0x174301106409cf95, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0716d3830}, 0xc05b814d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106409cf95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106409cf95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0716d37a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0716d37a0}, {0xc00a64f3f8, 0x660089}, 0xc00a64f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0716d37a0}, {0x7fe3f7399528}, 0xc05b814b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0716d3710}, 0xc05b814b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}, 0xc02cc6dfa0, 0xc08b141500) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}, 0x3, 0xc075c96b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a64f980}}}, {0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}, 0xc02cc6dfa0, 0xc075c96b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0716d3710}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0716d3710}, {0x40, 0xc00a64fb70}, 0xc00a64fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0716d3710}, {0x4e9d7a0, 0xc05b814b00}, 0xc02cc6dfa0, 0xc08b141500) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0716d3710}, 0xc050206d80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc087816120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc087816120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28230 [select, 6 minutes]: github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).run(0xc00cb29630, {0x63c85b8, 0xc02ca0c960}) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:159 +0x3ac github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).Start.func1({0x63c85b8, 0xc02ca0c960}) github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:101 +0x30 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 15946 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1({0x63c8510, 0xc002936240}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:116 +0x15a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219808572 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc06f852120) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0cd4ff340, {0x4d83cc0, 0xc06481ac80}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xcb) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0277ffe60, 0xc07a841178, 0xc07a841148) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0277ffe60, {0x4d83cc0, 0xc06481ac80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc042b86c90}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc050f94600}, {0x4d83cc0, 0xc06481ac80}, 0x2, {0xc00c7a1180, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc042b86c90}, {0x4f6406d, 0x7fe457d7ff18}, {0x4e9d7a0, 0xc050f94600}, {0x4d83cc0, 0xc06481ac80}, 0x4adf72d582048d4e, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x7c0c46e6d68644ab, {0x63c85b8, 0xc042b86c90}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc050f94600}, {0x4d83cc0, 0xc06481ac80}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc033a92ea8, {0x63c85b8, 0xc042b86c90}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc033a92ea8}}, {0x63c85b8, 0xc042b86c90}, 0xc050f94600, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1d, 0x1d, 0x12, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc08a8fc780, {0x63c85b8, 0xc042b869c0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc08a8fc780, {0x63c85b8, 0xc042b869c0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc018a34550, {0x63c85b8, 0xc042b869c0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc04f3a6000, {0x63c85b8, 0xc042b869c0}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc06f471080) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:264 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc050a07a40) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0a2b04000) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc050131130, {0x63c8510, 0xc050a079c0}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0393e3720, {0x63c8510, 0xc050a079c0}, 0xc0cd4fefd0) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc050a07780}, 0xc0d78850a0, 0xc0cd4fefd0, 0xc06481aa00, 0xc061a32e00, 0xc0e0dd55e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc050a07780}, 0xc0e0dd52f0, 0xc0d78850a0, 0xc0edd206c0, {{0x63ca3e8, 0xc06481a980}, 0x0}, 0xc061a32e00) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0e0dd4c00, {0x63c8510, 0xc050a07780}, 0xc0e0dd52f0, 0xc050a07780, {0x7fe42b790cf0, 0xc0edd206c0}, 0x0, 0xc04d73b978) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0e0dd4c00, {0x63c8510, 0xc050a07780}, 0xc0e0dd52f0, {0x7fe42b790cf0, 0xc0edd206c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0e0dd4c00, {0x63c85b8, 0xc042b86000}, {{0x63f8d78, 0xc0405dbcc0}, {0xc074554623, 0x3a}, 0x1, 0x1}, 0xc01db58900, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0e0dd4c00, {0x63c85b8, 0xc042b86000}, {{0x63f8d78, 0xc0405dbcc0}, {0xc074554623, 0x3a}, 0x1, 0x1}, 0xc01db58900, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0e0dd4c00, {0x63c85b8, 0xc042b86000}, {0xc01db58900, {0xc01dfe9d10, 0x1, 0x1}, {0xc014e4c4e0, 0x7, 0x7}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc074554b1a, 0x0}, 0x0, {0xc75fdaa, 0xedb7a8351, 0x0}, 0x1}, 0xc0e0dd4c00, 0xc07a847980, 0xc07a847970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0e0dd4c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0e0dd4c00, {0x63c8510, 0xc0085a0ac0}, 0xc07d13bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06c5d5480) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0085a0ac0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06c5d5480) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812309 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0310c8140, {0x63c85b8, 0xc0419dcc00}, 0xc040c55848, 0x1, 0xc01f7cb0a8, 0x519bbc, 0xc05063f740, 0xc05fe9e0c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc08fb50480, {0x63c85b8, 0xc0419dcc00}, 0xc05063f740, 0xc01f7cb1a8, 0x31, 0x40, 0x1, 0xc05063f740, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0310c8140, {0x63c85b8, 0xc0419dcc00}, 0xc05063f680, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc09fae0000, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0419dcc00}, 0x6352260, 0x655a2460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc03107e870, {0x63c85b8, 0xc0419dcc00}, {0xc05063f560, {0x1743011044d2d939, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc03107e870, {0x63c85b8, 0xc0419dcc00}, 0xc076ff4c80, 0xc01f7cb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0419dcc00}, 0x660089, {0xc05063f560, {0x1743011044d2d939, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310be400, {0x63c85b8, 0xc0419dcc00}, 0xc081b8f200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011044d2d939, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011044d2d939, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0419dcb70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0419dcb70}, {0xc01f7cd3f8, 0x660089}, 0xc01f7cd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0419dcb70}, {0x7fe32629fc10}, 0xc081b8f000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0419dcae0}, 0xc081b8f000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}, 0xc014d2b740, 0xc0434ce288) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}, 0x3, 0xc0609fe740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f7cd980}}}, {0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}, 0xc014d2b740, 0xc0609fe740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0419dcae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0419dcae0}, {0x40, 0xc01f7cdb70}, 0xc01f7cdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0419dcae0}, {0x4e9d7a0, 0xc081b8f000}, 0xc014d2b740, 0xc0434ce288) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0419dcae0}, 0xc02712eea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05063f440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05063f440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801011 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0894e4548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0894e45b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc090ba1bf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc020d2d880, {0xc017c79100, 0x2b, 0x79}, {0xc03ecca310, 0x2b, 0x6e}, 0x1, 0x1b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc090ba1a88, {0xc017c79100, 0x1, 0x6e}, {0xc03ecca310, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc090ba1a88, {0xc017c79100, 0x2b, 0x79}, {0xc03ecca310, 0x2b, 0x6e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc090ba11c0, {0xc03ecca310, 0x6e, 0x79}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc090ba11c0, {0xc017c79100, 0x2b, 0xc03ecca310}, {0xc03ecca310, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc090ba0f00, {0xc03ecca310, 0x2b, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc058ba8c08, {{0xc0aad07950, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0f15c4e00, {{0xc0aad07950, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc037df6c00, {0x63c85b8, 0xc044cca750}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc044cca750}, {0x64a6560, 0xc0f15c4e00}, {0xc0aad07950, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc044cca750}, {0x6466ed8, 0xc058ba8c00}, {0xc0aad07950, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02e7cf1e8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04b06c400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c03000, {0x63c85b8, 0xc044cca750}, 0xc04b06c400, 0xc057ce1720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c03000, {0x63c85b8, 0xc044cca750}, 0xc04b06c400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044cca6c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044cca6c0}, {0xc0562c48f0, 0x660089}, 0xc0562c48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044cca6c0}, {0xc00102e4e0}, 0xc04b06c300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044cca660}, 0xc04b06c300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a4ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219688196 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc03885cbb0, {0x63c85b8, 0xc07b5091a0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc03d6ef900, {0x63c85b8, 0xc07b5091a0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc03d6ef900, {0x63c8510, 0xc051704d80}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc03d6ef900, {0x63c8510, 0xc051704d80}, {0x63f7db8, 0x9bf6f48}, 0xc07dc6b088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc03d6ef900, {0x63c85b8, 0xc07747be00}, {{0x63f7db8, 0x9bf6f48}, {0xc037cb23de, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc03d6ef900, {0x63c85b8, 0xc07747be00}, {{0x63f7db8, 0x9bf6f48}, {0xc037cb23de, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc037cb23de, 0x6}, 0x0, 0x0}, {0x41a7268, 0xedb7a8350, 0x0}, {0x41a789f, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03d6ef900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03d6ef900, {0x63c8510, 0xc06fe1c3c0}, 0xc07de95dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03fabc2d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06fe1c3c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03fabc2d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 4223 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc0153e2ee0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc00b7e53c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}, 0xc015d8a348, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}, 0x3, 0xc01d11bd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0xc007d818c0}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}, 0xc015d8a348, 0xc01d11bd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01cc5de00}, {0x460ef4, 0xc002ca1bc8}, 0xc002ca1b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}, 0xc015d8a348, 0xc01d11bd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e31e0}, 0xc015d8a348, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc01979ab60}, 0xc0124e3320, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979ab60}, 0xc0124e3320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806748 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc012027400, {0x63c85b8, 0xc08de63ce0}, 0xc066038700, 0xc03ec75400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc012027400, {0x63c85b8, 0xc08de63ce0}, 0xc066038700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050afe9a5, 0x0, 0x0}, 0xc08d552f80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050afe9a5, 0x0, 0x0}, 0xc08d552f80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08de63c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08de63c50}, {0xc0936968f0, 0x660089}, 0xc0936968f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08de63c50}, {0xc00102e4e0}, 0xc066038600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08de63bf0}, 0xc066038600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc08e3fa300}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 17330 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99878, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00a757300, 0xc003b24000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00a757300, {0xc003b24000, 0x920, 0x920}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00a757300, {0xc003b24000, 0xc003b24005, 0x62}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0146f0738, {0xc003b24000, 0x6, 0xc002c9d788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc03735de60, {0xc003b24000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0036c8278, {0x631b900, 0xc03735de60}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0036c8000, {0x63290e0, 0xc0146f0738}, 0x2) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0036c8000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0036c8000, {0xc003b32000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc0190c7020, {0xc0197daba0, 0x9, 0xc03fe6f8f0}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc0190c7020}, {0xc0197daba0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0197daba0, 0x9, 0xc03fe6f830}, {0x631b600, 0xc0190c7020}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0197dab60) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00d112b40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 219801107 [semacquire]: sync.runtime_SemacquireMutex(0xc01352b140, 0xc0, 0x4f0948f) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc0185748e0) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose.func3() github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:285 +0x5c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc018574800, {0x63c85b8, 0xc0a51e1d70}, 0xc058268d00, 0xc03d1aa460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:318 +0xe12 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018574800, {0x63c85b8, 0xc0a51e1d70}, 0xc058268d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101623c7df, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101623c7df, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0a51e1b00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a51e1b00}, {0xc04de873f8, 0x660089}, 0xc04de87400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0a51e1b00}, {0x8801003d3d514c70}, 0xc058268a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0a51e1a70}, 0xc058268a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}, 0xc050f083a0, 0xc0599d1f68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}, 0x3, 0xc045fcfd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04de87980}}}, {0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}, 0xc050f083a0, 0xc045fcfd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0a51e1a70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a51e1a70}, {0x40, 0xc04de87b70}, 0xc04de87ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0a51e1a70}, {0x4e9d7a0, 0xc058268a00}, 0xc050f083a0, 0xc0599d1f68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0a51e1a70}, 0xc04dc455c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc01d32f440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc01d32f440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808645 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc0917af590}, 0xc0474915c0, 0x1, 0xc00f9490a8, 0x519bbc, 0xc037cedaa0, 0xc038479140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0917af590}, 0x1e0, 0xc00f949230, 0x37ceda40, 0x0, 0x660069, 0xc037cedaa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc0917af590}, 0xc037ceda40, {{{{0x0, 0x0}, {0xc05c8dd8c0, 0x53}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0917af590}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc0917af590}, {0xc060abeea0, {0x174301105d716183, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc0917af590}, 0xc05fa4ed20, 0xc00f9499f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0917af590}, 0x660089, {0xc060abeea0, {0x174301105d716183, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0917af590}, 0xc03f743000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105d716183, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105d716183, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0917af500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0917af500}, {0xc00f94b3f8, 0x660089}, 0xc00f94b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0917af500}, {0x40b08dfe1479981f}, 0xc03f742e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0917af440}, 0xc03f742e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}, 0xc0b88a4d80, 0xc0672e9158) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}, 0x3, 0xc04552c240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f94b980}}}, {0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}, 0xc0b88a4d80, 0xc04552c240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0917af440}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0917af440}, {0x40, 0xc00f94bb70}, 0xc00f94bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0917af440}, {0x4e9d7a0, 0xc03f742e00}, 0xc0b88a4d80, 0xc0672e9158) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0917af440}, 0xc029af9860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060abed80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060abed80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812181 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc068600cd0, {0x63c85b8, 0xc03cbf1410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc03cbf1410}, {0x4f67ad2, 0x6446ff8}, 0xc07e11cfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 28394 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1({0x63c8510, 0xc0203a8940}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:116 +0x15a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219811149 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0473c4c30}, 0xc0159ba978, 0x1, 0xc02b62b0a8, 0x519bbc, 0xc073ea7e60, 0xc073ea7aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0473c4c30}, 0x1e0, 0xc02b62b230, 0x73ea7e00, 0x0, 0x660069, 0xc073ea7e60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0473c4c30}, 0xc073ea7e00, {{{{0x0, 0x0}, {0xc0504e5440, 0x61}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0473c4c30}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0473c4c30}, {0xc0308d6360, {0x1743011074a011c7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0473c4c30}, 0xc0496fde00, 0xc02b62b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0473c4c30}, 0x660089, {0xc0308d6360, {0x1743011074a011c7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0473c4c30}, 0xc045e94a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011074a011c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011074a011c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0473c4ba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473c4ba0}, {0xc02b62d3f8, 0x660089}, 0xc02b62d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0473c4ba0}, {0x7441316736303230}, 0xc045e94600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0473c4b10}, 0xc045e94600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}, 0xc01ed83e00, 0xc06292f200) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}, 0x3, 0xc03763b240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02b62d980}}}, {0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}, 0xc01ed83e00, 0xc03763b240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0473c4b10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473c4b10}, {0x40, 0xc02b62db70}, 0xc02b62dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0473c4b10}, {0x4e9d7a0, 0xc045e94600}, 0xc01ed83e00, 0xc06292f200) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0473c4b10}, 0xc065e1cae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0308d6120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0308d6120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 29519 [select, 654 minutes]: github.com/cockroachdb/cockroach/pkg/server/diagnostics.(*UpdateChecker).PeriodicallyCheckForUpdates.func1({0x63c85b8, 0xc02b3c3c80}) github.com/cockroachdb/cockroach/pkg/server/diagnostics/update_checker.go:94 +0x2c5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802861 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc03fe76d20}, 0xc029098e58, 0x1, 0xc0683dd0a8, 0x519bbc, 0xc040d527e0, 0xc040912d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03fe76d20}, 0x1e0, 0xc0683dd230, 0x40d52780, 0x0, 0x660069, 0xc040d527e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc03fe76d20}, 0xc040d52780, {{{{0x0, 0x0}, {0xc028415680, 0x3d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x4, {0x63c85b8, 0xc03fe76d20}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc03fe76d20}, {0xc05056b560, {0x174301102488145a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc03fe76d20}, 0xc046df3f40, 0xc0683dd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03fe76d20}, 0x660089, {0xc05056b560, {0x174301102488145a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc03fe76d20}, 0xc08b42d500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102488145a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102488145a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03fe76c30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fe76c30}, {0xc0683df3f8, 0x660089}, 0xc0683df400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03fe76c30}, {0x0}, 0xc08b42d300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03fe76ba0}, 0xc08b42d300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}, 0xc084769f60, 0xc015eddb90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}, 0x3, 0xc05c7c7ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0683df980}}}, {0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}, 0xc084769f60, 0xc05c7c7ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03fe76ba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fe76ba0}, {0x40, 0xc0683dfb70}, 0xc0683dfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03fe76ba0}, {0x4e9d7a0, 0xc08b42d300}, 0xc084769f60, 0xc015eddb90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03fe76ba0}, 0xc02cb0bb00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05056b440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05056b440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811742 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b85f680, {0x63c85b8, 0xc06f283fb0}, 0xc0450fcd38, 0x1, 0x7080181914089e, 0x37e11d600, 0xc06275f4a0, 0xc0851e6ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06f283fb0}, 0x120, 0xc0335f9230, 0x6275f440, 0x0, 0x660069, 0xc06275f4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b85f680, {0x63c85b8, 0xc06f283fb0}, 0xc06275f440, {{{{0x0, 0x0}, {0xc051fec5a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06f283fb0}, 0x6352410, 0x3abf6080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b7d3a40, {0x63c85b8, 0xc06f283fb0}, {0xc06275f320, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b7d3a40, {0x63c85b8, 0xc06f283fb0}, 0xc070e27d60, 0xc0335f99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f283fb0}, 0x660089, {0xc06275f320, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b922c00, {0x63c85b8, 0xc06f283fb0}, 0xc082c3e500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f283ec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f283ec0}, {0xc0335fb3f8, 0x660089}, 0xc0335fb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f283ec0}, {0x0}, 0xc082c3e300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f283e30}, 0xc082c3e300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}, 0xc0392940c0, 0xc0450fcd08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}, 0x3, 0xc06566d200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0335fb980}}}, {0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}, 0xc0392940c0, 0xc06566d200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f283e30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f283e30}, {0x40, 0xc0335fbb70}, 0xc0335fbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f283e30}, {0x4e9d7a0, 0xc082c3e300}, 0xc0392940c0, 0xc0450fcd08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f283e30}, 0xc08f62db60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc017662a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc017662a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809284 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc07b6c18c0}, 0xc07cd7ac00, 0xc057dde000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc07b6c18c0}, 0xc07cd7ac00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110641a6045, 0x0, 0x0}, 0xc078d38a60, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110641a6045, 0x0, 0x0}, 0xc078d38a60, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07b6c1830}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b6c1830}, {0xc0127f53f8, 0x660089}, 0xc0127f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07b6c1830}, {0x0}, 0xc07cd7a900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07b6c1770}, 0xc07cd7a900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}, 0xc050616a60, 0xc0246f9668) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}, 0x3, 0xc00e4e2780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0127f5980}}}, {0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}, 0xc050616a60, 0xc00e4e2780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07b6c1770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b6c1770}, {0x40, 0xc0127f5b70}, 0xc0127f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07b6c1770}, {0x4e9d7a0, 0xc07cd7a900}, 0xc050616a60, 0xc0246f9668) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07b6c1770}, 0xc085497ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04c55ac60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04c55ac60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803625 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f6e83c0, {0x63c85b8, 0xc05465c8a0}, 0xc03e974d08, 0x1, 0xc04f56f0a8, 0x519bbc, 0xc04141e600, 0xc0402fd020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05465c8a0}, 0x120, 0xc04f56f230, 0x4141e5a0, 0x0, 0x660069, 0xc04141e600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f6e83c0, {0x63c85b8, 0xc05465c8a0}, 0xc04141e5a0, {{{{0x0, 0x0}, {0xc079915c20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05465c8a0}, 0x6352410, 0x3080fe90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f6dc320, {0x63c85b8, 0xc05465c8a0}, {0xc04141e480, {0x174301102e4959b8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f6dc320, {0x63c85b8, 0xc05465c8a0}, 0xc01858bae0, 0xc04f56f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05465c8a0}, 0x660089, {0xc04141e480, {0x174301102e4959b8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f6ec400, {0x63c85b8, 0xc05465c8a0}, 0xc04b2e3000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e4959b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e4959b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05465c810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05465c810}, {0xc04f5713f8, 0x660089}, 0xc04f571400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05465c810}, {0x7fe3bc093740}, 0xc04b2e2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05465c780}, 0xc04b2e2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}, 0xc07a62bfa0, 0xc02060fb48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}, 0x3, 0xc011fd4e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04f571980}}}, {0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}, 0xc07a62bfa0, 0xc011fd4e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05465c780}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05465c780}, {0x40, 0xc04f571b70}, 0xc04f571ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05465c780}, {0x4e9d7a0, 0xc04b2e2e00}, 0xc07a62bfa0, 0xc02060fb48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05465c780}, 0xc02bf5e360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04141e360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04141e360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807013 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc04a607b60}, 0xc067797d40, 0x1, 0xc00515f0a8, 0x519bbc, 0xc09a53d4a0, 0xc081f8a840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04a607b60}, 0x120, 0xc00515f230, 0x9a53d440, 0x0, 0x660069, 0xc09a53d4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc04a607b60}, 0xc09a53d440, {{{{0x0, 0x0}, {0xc082ccd200, 0x60}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04a607b60}, 0x6352410, 0x5449b8b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc04a607b60}, {0xc09a53d320, {0x1743011042c871a8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc04a607b60}, 0xc0b7068be0, 0xc00515f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04a607b60}, 0x660089, {0xc09a53d320, {0x1743011042c871a8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc04a607b60}, 0xc07a599000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011042c871a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011042c871a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04a607ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a607ad0}, {0xc0051613f8, 0x660089}, 0xc005161400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04a607ad0}, {0x0}, 0xc07a598d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04a607a40}, 0xc07a598d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}, 0xc049100240, 0xc045b8d8a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}, 0x3, 0xc0528657c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005161980}}}, {0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}, 0xc049100240, 0xc0528657c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04a607a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a607a40}, {0x40, 0xc005161b70}, 0xc005161ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04a607a40}, {0x4e9d7a0, 0xc07a598d00}, 0xc049100240, 0xc045b8d8a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04a607a40}, 0xc060470180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc034eb9680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc034eb9680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802825 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc02211ea50}, 0xc026690b70, 0x1, 0xc064b930a8, 0x519bbc, 0xc03e9d69c0, 0xc039e63320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02211ea50}, 0x1e0, 0xc064b93230, 0x3e9d6960, 0x0, 0x660069, 0xc03e9d69c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc02211ea50}, 0xc03e9d6960, {{{{0x0, 0x0}, {0xc051f5d680, 0x36}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02211ea50}, 0x63525f0, 0x25eada78) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc02211ea50}, {0xc04934e240, {0x1743011024123859, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02211ea50}, 0xc03b3659a0, 0xc064b939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02211ea50}, 0x660089, {0xc04934e240, {0x1743011024123859, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02211ea50}, 0xc0aebea900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024123859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024123859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02211e990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02211e990}, {0xc064b953f8, 0x660089}, 0xc064b95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02211e990}, {0x0}, 0xc0aebea700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02211e900}, 0xc0aebea700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}, 0xc086683ea0, 0xc01e0e13e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}, 0x3, 0xc036e3d280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc064b95980}}}, {0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}, 0xc086683ea0, 0xc036e3d280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02211e900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02211e900}, {0x40, 0xc064b95b70}, 0xc064b95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02211e900}, {0x4e9d7a0, 0xc0aebea700}, 0xc086683ea0, 0xc01e0e13e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02211e900}, 0xc01c8253e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc036210360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc036210360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812786 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803410 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc060f206c0}, 0xc063e53f00, 0xc0686f5860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc060f206c0}, 0xc063e53f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bed681e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bed681e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060f20630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060f20630}, {0xc0761d93f8, 0x660089}, 0xc0761d9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060f20630}, {0x3}, 0xc063e53b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc060f205a0}, 0xc063e53b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}, 0xc0382988e0, 0xc070738078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}, 0x3, 0xc045e77740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0761d9980}}}, {0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}, 0xc0382988e0, 0xc045e77740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc060f205a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060f205a0}, {0x40, 0xc0761d9b70}, 0xc0761d9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc060f205a0}, {0x4e9d7a0, 0xc063e53b00}, 0xc0382988e0, 0xc070738078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc060f205a0}, 0xc04adeafc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03690e7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03690e7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28269 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc003583cf8) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01d230540) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01d230540, {0x63c8510, 0xc02aec1080}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00167cf00, {0x63c8510, 0xc02aec1040}, {0xc02ccd8db0, 0x1, 0xc001b00f28}, {0x1cf6ee7, 0xd3ab40, 0xc0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:139 +0x625 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc001b00f10, {0x63c8510, 0xc02aec1040}, {0xc02ccd8db0, 0xc001b00f98, 0x1893c7b}, {0x707000003d913e4, 0x63c85b8, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:78 +0x45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x63c8510, 0xc02aec1040}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:285 +0x7a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219738072 [IO wait]: internal/poll.runtime_pollWait(0x7fdb124bca90, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0657da000, 0xc03615c000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0657da000, {0xc03615c000, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0657da000, {0xc03615c000, 0x739, 0xc058297b80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc04831cee8, {0xc03615c000, 0x7eda79, 0xc03615c005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01c47f500, {0xc03615c000, 0x5, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0488cd8c0, {0xc03615c000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0906ac5f8, {0x631b900, 0xc0488cd8c0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0906ac380, {0x7fe42ab59310, 0xc01c47f500}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0906ac380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0906ac380, {0xc01b3f7000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0b3d4ab70, {0xc01b3f7000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc077aa7588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc077aa7588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc077aa7b60, {0x634aac0, 0xc077aa7588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc077aa7500, 0xc06c0d01e0, 0xc0081e342b, 0xc0081e342d, {0x635bec0, 0xc026a49d40}, 0xc000d6ae00, 0xc015da8a58, 0xc0081e3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc077aa7500, {0x63c85b8, 0xc04dc2bc50}, 0xc06c0d01d0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc05015e043, 0xb}}, 0x0, 0xc04dc2bbf0, 0xc04dc2bc20, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04dc2bc50}, {0x643d810, 0xc01c47f500}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc040f4a780}, {0x643d810, 0xc01c47f500}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc040f4a780}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219784480 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b29a0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc077d8bf00, 0xc077179800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc077d8bf00, {0xc077179800, 0x73d, 0x73d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc077d8bf00, {0xc077179800, 0x73d, 0xc07c46a280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01abc4cc0, {0xc077179800, 0x7eda79, 0xc077179805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0cd844300, {0xc077179800, 0x7fdbb01bbfff, 0x73d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc039bdbbd8, {0xc077179800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0733b3778, {0x631b900, 0xc039bdbbd8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0733b3500, {0x7fe42ab59310, 0xc0cd844300}, 0x73d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0733b3500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0733b3500, {0xc08ea56000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc079aff998, {0xc08ea56000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0910c4b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0910c4b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0910c50e0, {0x634aac0, 0xc0910c4b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0910c4a80, 0xc07cc224d0, 0xc092c2942b, 0xc092c2942d, {0x635bec0, 0xc0765f6870}, 0xc000d6ae00, 0xc07a8ea9b8, 0xc092c29429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0910c4a80, {0x63c85b8, 0xc07a8c3f80}, 0xc07cc224c0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc03d9a501a, 0x10}}, 0x0, 0xc07a8c3f20, 0xc07a8c3f50, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc07a8c3f80}, {0x643d810, 0xc0cd844300}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07b584e70}, {0x643d810, 0xc0cd844300}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07b584e70}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 217838099 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaffb30b0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07284f300, 0xc00fdb9000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07284f300, {0xc00fdb9000, 0x2d97, 0x2d97}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07284f300, {0xc00fdb9000, 0x2d97, 0xc06507adc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03eaa1f70, {0xc00fdb9000, 0x7eda79, 0xc00fdb9005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0753fca20, {0xc00fdb9000, 0x0, 0x2d97}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0668d13c8, {0xc00fdb9000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc09dc8ecf8, {0x631b900, 0xc0668d13c8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc09dc8ea80, {0x7fe42ab59310, 0xc0753fca20}, 0x2d97) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc09dc8ea80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc09dc8ea80, {0xc0623bb000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc033001f38, {0xc0623bb000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a0512b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a0512b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a05130e0, {0x634aac0, 0xc0a0512b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a0512a80, 0xc021a75f20, 0xc053b7742b, 0xc053b7742d, {0x635bec0, 0xc00ef10f30}, 0xc000d6ae00, 0xc03b2de7f4, 0xc053b77429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a0512a80, {0x63c85b8, 0xc029ff52f0}, 0xc021a75f10, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc079c67021, 0xb}}, 0x0, 0xc029ff5290, 0xc029ff52c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc029ff52f0}, {0x643d810, 0xc0753fca20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04219ca50}, {0x643d810, 0xc0753fca20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04219ca50}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219788862 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc037bda000}, 0xc031b20cb0, {0x646cc48, 0xc037bda0c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc037bda000}, 0xedb7a834d, 0xc052f945a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc037bda000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc062c31c50}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc031b20ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc062c31c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 216537367 [sync.Cond.Wait, 5 minutes]: sync.runtime_notifyListWait(0xc0036dd510, 0x4) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0a943a000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0a943a660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06864ac00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06864ac00, {0x63c8510, 0xc0036dd540}, 0xc05164bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0580d9180) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0036dd540}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0580d9180) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809032 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fc5cc80, {0x63c85b8, 0xc030534a80}, 0xc04784bd28, 0x1, 0xc002b070a8, 0x519bbc, 0xc07a667800, 0xc082ccf500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0486f8b40, {0x63c85b8, 0xc030534a80}, 0xc0486f8b70, 0xc002b071a8, 0x18, 0x18, 0xffffffffffffffff, 0xc07a667800, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fc5cc80, {0x63c85b8, 0xc030534a80}, 0xc07a6677a0, {{{{0x0, 0x0}, {0xc060492240, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc030534a80}, 0x6352230, 0x47996540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fc6a500, {0x63c85b8, 0xc030534a80}, {0xc07a667680, {0x17430110635039d4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fc6a500, {0x63c85b8, 0xc030534a80}, 0xc065741e00, 0xc002b079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc030534a80}, 0x660089, {0xc07a667680, {0x17430110635039d4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fc83800, {0x63c85b8, 0xc030534a80}, 0xc04a569b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110635039d4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110635039d4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0305349f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0305349f0}, {0xc002b093f8, 0x660089}, 0xc002b09400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0305349f0}, {0x7fe426bc7510}, 0xc04a569900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc030534900}, 0xc04a569900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}, 0xc06acba4c0, 0xc0293e5d58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}, 0x3, 0xc02232c440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002b09980}}}, {0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}, 0xc06acba4c0, 0xc02232c440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc030534900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030534900}, {0x40, 0xc002b09b70}, 0xc002b09ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc030534900}, {0x4e9d7a0, 0xc04a569900}, 0xc06acba4c0, 0xc0293e5d58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc030534900}, 0xc078843ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07a667320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07a667320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28270 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00148f760, {0x63c8510, 0xc02aec1040}, 0xc02ccd8ae0, 0xc01ea0f3e0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:331 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x63c8510, 0xc02aec1040}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:288 +0x38 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219812794 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801043 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc061dce000, {0x63c85b8, 0xc02046e720}, 0xc070acde00, 0xc0307a6460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc02046e720}, 0xc070acde00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fdc409ece, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fdc409ece, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02046e4e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02046e4e0}, {0xc04dd9d3f8, 0x660089}, 0xc04dd9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02046e4e0}, {0x3fc3333333333333}, 0xc070acdc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02046e3f0}, 0xc070acdc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}, 0xc04cec37c0, 0xc0354c6c30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}, 0x3, 0xc050c07cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04dd9d980}}}, {0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}, 0xc04cec37c0, 0xc050c07cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02046e3f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02046e3f0}, {0x40, 0xc04dd9db70}, 0xc04dd9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02046e3f0}, {0x4e9d7a0, 0xc070acdc00}, 0xc04cec37c0, 0xc0354c6c30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02046e3f0}, 0xc07915c3c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03a44dd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03a44dd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812504 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc05b456dc0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc098985500, 0xc085440010) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc05b456dc0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc098985500, {0xc07d208bd0, 0x29, 0x6d}, {0xc03c449830, 0x29, 0x29}, 0x0, 0x1b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc076ea1a38, {0xc07d208bd0, 0x1, 0x29}, {0xc03c449830, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc076ea1a38, {0xc07d208bd0, 0x29, 0x6d}, {0xc03c449830, 0x29, 0x29}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc076ea0cc0, {0xc03c449830, 0x29, 0x6d}, 0x29, 0x1f) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc076ea0cc0, {0xc07d208bd0, 0x29, 0xc03c449830}, {0xc03c449830, 0x28, 0x50}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc076ea0a00, {0xc03c449830, 0x29, 0x29}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05879c028, {{0xc0475cd8b0, 0x28, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc062b2a100, {{0xc0475cd8b0, 0x28, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x64a6560, 0xc062b2a100}, {{0xc0475cd8b0, 0x28, 0x50}, {0x0, 0x0, 0x0}}, 0x80, 0xc08642d680) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:856 +0xbf github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal({0x63c85b8, 0xc04ae6b410}, {0x7fe42e217bc0, 0xc034107640}, {0x64a6560, 0xc062b2a100}, 0xc020024100, {0xc0475cd8b0, 0x28, 0x50}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1287 +0x13d github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter({0x63c85b8, 0xc04ae6b410}, {0x7fe42e217bc0, 0xc034107640}, {0x64a6560, 0xc062b2a100}, 0x0, {0xc0475cd8b0, 0x28, 0x50}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1073 +0x1cc github.com/cockroachdb/cockroach/pkg/storage.mvccInitPutUsingIter({0x63c85b8, 0xc04ae6b410}, {0x7fe42e217a50, 0xc034107640}, {0x64a6560, 0xc062b2a100}, 0x4c, {0xc0475cd8b0, 0x28, 0x50}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1849 +0x1a5 github.com/cockroachdb/cockroach/pkg/storage.MVCCInitPut({0x63c85b8, 0xc04ae6b410}, {0x7fe42e217a50, 0xc034107640}, 0xc052f3d490, {0xc0475cd8b0, 0x28, 0x50}, {0x1743011082f2a274, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1814 +0x26a github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.InitPut({_, _}, {_, _}, {{0x64d2498, 0xc02532b400}, {{0x1743011082f2a274, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_init_put.go:39 +0x254 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011082f2a274, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:493 +0x287 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05707ee00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:644 +0x190 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:612 +0x2da github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:428 +0x5f3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc02532b400, {0x63c85b8, 0xc04ae6b410}, {0xc06066c2e8, 0x8}, 0xc05707ee00, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:673 +0x1c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x0, {0x63c85b8, _}, {_, _}, _, {{{0x1742da323ca17200, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:773 +0xaa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc02532b400, {0x63c85b8, _}, _, _, {{{0x1742da323ca17200, 0x0, 0x0}, 0x0, {0x1f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:113 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02532b400, {0x63c85b8, 0xc04ae6b410}, 0xc05707ee00, 0xc0436ced20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:167 +0x7b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc04ae6b410}, 0xc05707ee00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ae6b380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ae6b380}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ae6b380}, {0xc00102e4e0}, 0xc05707ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ae6b320}, 0xc05707ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc018086ff0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802684 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01ed76400, {0x63c85b8, 0xc086bba270}, 0xc04c765c00, 0xc0686ee000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ed76400, {0x63c85b8, 0xc086bba270}, 0xc04c765c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110247ffe70, 0x0, 0x0}, 0xc0572a9d10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110247ffe70, 0x0, 0x0}, 0xc0572a9d10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086bba1e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086bba1e0}, {0xc05ac9b3f8, 0x660089}, 0xc05ac9b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086bba1e0}, {0x7fe33bf74258}, 0xc04c765a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086bba150}, 0xc04c765a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}, 0xc0270b5bc0, 0xc042d4a528) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}, 0x3, 0xc01921bf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05ac9b980}}}, {0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}, 0xc0270b5bc0, 0xc01921bf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086bba150}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086bba150}, {0x40, 0xc05ac9bb70}, 0xc05ac9bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086bba150}, {0x4e9d7a0, 0xc04c765a00}, 0xc0270b5bc0, 0xc042d4a528) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086bba150}, 0xc0228b96e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cb40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cb40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812789 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28126 [select]: github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1({0x63c85b8, 0xc02ca0c360}) github.com/cockroachdb/cockroach/pkg/sql/stats/pkg/sql/stats/automatic_stats.go:374 +0x1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 207646347 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3e0130, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06c5f2e80, 0xc027f2c000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06c5f2e80, {0xc027f2c000, 0x74e, 0x74e}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06c5f2e80, {0xc027f2c000, 0x74e, 0xc0683f9b80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc033df5308, {0xc027f2c000, 0x7eda79, 0xc027f2c005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01c21e7e0, {0xc027f2c000, 0x9, 0x74e}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc05bebfae8, {0xc027f2c000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc046480978, {0x631b900, 0xc05bebfae8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc046480700, {0x7fe42ab59310, 0xc01c21e7e0}, 0x74e) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc046480700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc046480700, {0xc01912d000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0720d0ba0, {0xc01912d000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc087b30b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc087b30b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc087b310e0, {0x634aac0, 0xc087b30b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc087b30a80, 0xc02dcbd0f0, 0xc064c3542b, 0xc064c3542d, {0x635bec0, 0xc01401aa20}, 0xc000d6ae00, 0xc030da65e4, 0xc064c35429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc087b30a80, {0x63c85b8, 0xc073bc6450}, 0xc02dcbd0e0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc076f14028, 0x21}}, 0x0, 0xc073bc63f0, 0xc073bc6420, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc073bc6450}, {0x643d810, 0xc01c21e7e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc064e23890}, {0x643d810, 0xc01c21e7e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc064e23890}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803294 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ef6c500, {0x63c85b8, 0xc05eae3980}, 0xc06508d3e0, 0x1, 0xc043a0f0a8, 0x519bbc, 0xc036e49ec0, 0xc060c1dc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05eae3980}, 0x120, 0xc043a0f230, 0x36e49e60, 0x0, 0x660069, 0xc036e49ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ef6c500, {0x63c85b8, 0xc05eae3980}, 0xc036e49e60, {{{{0x0, 0x0}, {0xc0470647e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05eae3980}, 0x6352410, 0x210ac018) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ef760a0, {0x63c85b8, 0xc05eae3980}, {0xc036e49d40, {0x17430110292b3b32, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ef760a0, {0x63c85b8, 0xc05eae3980}, 0xc0c36a92c0, 0xc043a0f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05eae3980}, 0x660089, {0xc036e49d40, {0x17430110292b3b32, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ef64800, {0x63c85b8, 0xc05eae3980}, 0xc0f1580700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110292b3b32, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110292b3b32, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05eae3890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05eae3890}, {0xc043a113f8, 0x660089}, 0xc043a11400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05eae3890}, {0xc0015221e0}, 0xc0f1580500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05eae3800}, 0xc0f1580500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}, 0xc06e7ccb80, 0xc0592bbf98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}, 0x3, 0xc02889e540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043a11980}}}, {0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}, 0xc06e7ccb80, 0xc02889e540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05eae3800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05eae3800}, {0x40, 0xc043a11b70}, 0xc043a11ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05eae3800}, {0x4e9d7a0, 0xc0f1580500}, 0xc06e7ccb80, 0xc0592bbf98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05eae3800}, 0xc0787b7980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc036e49c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc036e49c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812102 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc0190e8780}, 0xc0ce7a5128, 0x1, 0xc0235cf0a8, 0x519bbc, 0xc06bac3ce0, 0xc05ad5a9c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05adacea0, {0x63c85b8, 0xc0190e8780}, 0xc05adacfa0, 0xc0235cf1a8, 0x26, 0x40, 0xffffffffffffffff, 0xc06bac3ce0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc0190e8780}, 0xc06bac3c20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc059238900, 0x46}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0190e8780}, 0x6352260, 0x655a2240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc0190e8780}, {0xc06bac3b00, {0x1743011041ec4a1c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc0190e8780}, 0xc052babf40, 0xc0235cf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0190e8780}, 0x660089, {0xc06bac3b00, {0x1743011041ec4a1c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0190e8780}, 0xc07afe2500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041ec4a1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041ec4a1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0190e86c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0190e86c0}, {0xc0235d13f8, 0x660089}, 0xc0235d1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0190e86c0}, {0xc0015221e0}, 0xc07afe2300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0190e8630}, 0xc07afe2300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}, 0xc024b384e0, 0xc0ce7a5ab8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}, 0x3, 0xc0109dd380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0235d1980}}}, {0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}, 0xc024b384e0, 0xc0109dd380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0190e8630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0190e8630}, {0x40, 0xc0235d1b70}, 0xc0235d1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0190e8630}, {0x4e9d7a0, 0xc07afe2300}, 0xc024b384e0, 0xc0ce7a5ab8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0190e8630}, 0xc0466feea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06bac39e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06bac39e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805749 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc000c98800, {0x63c85b8, 0xc0835252c0}, 0xc04f744e00, 0xc0831fe780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000c98800, {0x63c85b8, 0xc0835252c0}, 0xc04f744e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110480a45c8, 0x0, 0x0}, 0xc02816aad0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110480a45c8, 0x0, 0x0}, 0xc02816aad0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc083525230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc083525230}, {0xc004e593f8, 0x660089}, 0xc004e59400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc083525230}, {0x8e3c6058023786dc}, 0xc04f744b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0835251a0}, 0xc04f744b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}, 0xc06f237b00, 0xc026b79c68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}, 0x3, 0xc06e0fe780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004e59980}}}, {0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}, 0xc06f237b00, 0xc06e0fe780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0835251a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0835251a0}, {0x40, 0xc004e59b70}, 0xc004e59ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0835251a0}, {0x4e9d7a0, 0xc04f744b00}, 0xc06f237b00, 0xc026b79c68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0835251a0}, 0xc044d35aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093732240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093732240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810035 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc019ecf650}, 0xc02febf068, 0x1, 0xc0289310a8, 0x519bbc, 0xc00aa9d4a0, 0xc0c310a2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc019ecf650}, 0x120, 0xc028931230, 0xaa9d440, 0x0, 0x660069, 0xc00aa9d4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc019ecf650}, 0xc00aa9d440, {{{{0x0, 0x0}, {0xc05112d440, 0xf5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc019ecf650}, 0x6352410, 0x43a28848) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc019ecf650}, {0xc00aa9d200, {0x17430110692a12a4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc019ecf650}, 0xc052bab9a0, 0xc0289319f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc019ecf650}, 0x660089, {0xc00aa9d200, {0x17430110692a12a4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc019ecf650}, 0xc08b935e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110692a12a4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110692a12a4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc019ecf5c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019ecf5c0}, {0xc0289333f8, 0x660089}, 0xc028933400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc019ecf5c0}, {0xc0015221e0}, 0xc08b935c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc019ecf530}, 0xc08b935c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}, 0xc01e9ef0c0, 0xc02febf9f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}, 0x3, 0xc00844dbc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028933980}}}, {0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}, 0xc01e9ef0c0, 0xc00844dbc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc019ecf530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019ecf530}, {0x40, 0xc028933b70}, 0xc028933ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc019ecf530}, {0x4e9d7a0, 0xc08b935c00}, 0xc01e9ef0c0, 0xc02febf9f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc019ecf530}, 0xc021786ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00aa9cfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00aa9cfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803928 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030065c00, {0x63c85b8, 0xc04d8a1920}, 0xc0d46e8100, 0xc06891a280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc04d8a1920}, 0xc0d46e8100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103102b30a, 0x0, 0x0}, 0xc037f46b40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103102b30a, 0x0, 0x0}, 0xc037f46b40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d8a1890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d8a1890}, {0xc00592f3f8, 0x660089}, 0xc00592f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d8a1890}, {0x7fe32637ada8}, 0xc070567e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d8a1800}, 0xc070567e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}, 0xc085621100, 0xc0b43ad5a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}, 0x3, 0xc06d189a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00592f980}}}, {0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}, 0xc085621100, 0xc06d189a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d8a1800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d8a1800}, {0x40, 0xc00592fb70}, 0xc00592fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d8a1800}, {0x4e9d7a0, 0xc070567e00}, 0xc085621100, 0xc0b43ad5a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d8a1800}, 0xc05b148960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04dd67680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04dd67680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809834 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020b43cc0, {0x63c85b8, 0xc06422e960}, 0xc020c84db0, 0x1, 0xc0058d30a8, 0x519bbc, 0xc07b8aacc0, 0xc08231cf00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06422e960}, 0x120, 0xc0058d3230, 0x7b8aac60, 0x0, 0x660069, 0xc07b8aacc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020b43cc0, {0x63c85b8, 0xc06422e960}, 0xc07b8aac60, {{{{0x0, 0x0}, {0xc0d6108a20, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06422e960}, 0x6352410, 0x6bab3ff8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc020bfeb90, {0x63c85b8, 0xc06422e960}, {0xc07b8aab40, {0x1743011067081cd9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc020bfeb90, {0x63c85b8, 0xc06422e960}, 0xc04d955c20, 0xc0058d39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06422e960}, 0x660089, {0xc07b8aab40, {0x1743011067081cd9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0216b8400, {0x63c85b8, 0xc06422e960}, 0xc04f22bb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067081cd9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067081cd9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06422e8d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06422e8d0}, {0xc0058d53f8, 0x660089}, 0xc0058d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06422e8d0}, {0x0}, 0xc04f22b900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06422e810}, 0xc04f22b900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}, 0xc02af0d3a0, 0xc020c85848) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}, 0x3, 0xc06537dac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0058d5980}}}, {0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}, 0xc02af0d3a0, 0xc06537dac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06422e810}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06422e810}, {0x40, 0xc0058d5b70}, 0xc0058d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06422e810}, {0x4e9d7a0, 0xc04f22b900}, 0xc02af0d3a0, 0xc020c85848) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06422e810}, 0xc043652720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc062dbc240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc062dbc240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791443 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02e27c8a0}, 0xc0918d5cb0, {0x646cc48, 0xc02e27c960}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02e27c8a0}, 0xedb7a834e, 0xc08d27ec60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02e27c8a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc049628e10}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0918d5ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc049628e10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219141870 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc06da0be00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 219804539 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021d55680, {0x63c85b8, 0xc02b67cb10}, 0xc07b4d0ee8, 0x1, 0xc07689f0a8, 0x519bbc, 0xc061f87c80, 0xc06060d4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02b67cb10}, 0x120, 0xc07689f230, 0x61f87c20, 0x0, 0x660069, 0xc061f87c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021d55680, {0x63c85b8, 0xc02b67cb10}, 0xc061f87c20, {{{{0x0, 0x0}, {0xc0750c4fc0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02b67cb10}, 0x6352410, 0x17e2e7c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3cdc0, {0x63c85b8, 0xc02b67cb10}, {0xc061f87b00, {0x1743011034452c8e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3cdc0, {0x63c85b8, 0xc02b67cb10}, 0xc083df46e0, 0xc07689f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02b67cb10}, 0x660089, {0xc061f87b00, {0x1743011034452c8e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021dac400, {0x63c85b8, 0xc02b67cb10}, 0xc0618db600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011034452c8e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011034452c8e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b67c900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b67c900}, {0xc0768a13f8, 0x660089}, 0xc0768a1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b67c900}, {0x0}, 0xc0618db300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02b67c690}, 0xc0618db300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}, 0xc073550a00, 0xc042fbb7d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}, 0x3, 0xc01605e380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0768a1980}}}, {0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}, 0xc073550a00, 0xc01605e380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02b67c690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b67c690}, {0x40, 0xc0768a1b70}, 0xc0768a1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02b67c690}, {0x4e9d7a0, 0xc0618db300}, 0xc073550a00, 0xc042fbb7d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02b67c690}, 0xc0cdae8240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc061f879e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc061f879e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28487 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00ad46540, {0x63c8510, 0xc0206eca00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:310 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00ad46540, {0x63c85b8, 0xc01cbda900}, 0xc011ffec00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:335 +0xbb github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x63c85b8, 0xc01cbda900}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801869 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7b80, {0x63c85b8, 0xc07956a1b0}, 0xc0c5888f00, 0x1, 0xc06102dff0, 0x519bbc, 0xc0b1af58e0, 0xc0ae37d8e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x360, {0x63c85b8, 0xc07956a1b0}, 0x380, 0xc06102e178, 0xb1af5880, 0x0, 0x660069, 0xc0b1af58e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7b80, {0x63c85b8, 0xc07956a1b0}, 0xc0b1af5880, {{{{0x0, 0x0}, {0xc09f8e0a20, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07956a1b0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2910, {0x63c85b8, 0xc07956a1b0}, {0xc053ec8000, {0x174301101e559fd2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2910, {0x63c85b8, 0xc07956a1b0}, 0xc03d6df720, 0xc04c2f2938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07956a1b0}, 0x660089, {0xc053ec8000, {0x174301101e559fd2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0fb800, {0x63c85b8, 0xc07956a1b0}, 0xc06bec9100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101e559fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101e559fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07956a090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07956a090}, {0xc04c2f4340, 0x660089}, 0xc04c2f4348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07956a090}, {0xc00102e4e0}, 0xc06bec9000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03be45f80}, 0xc06bec9000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc03be45b90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219663696 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0b813a810, 0x51) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc01278a000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc01278a660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0166ab900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0166ab900, {0x63c8510, 0xc0b813a840}, 0xc058167dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc076bb32f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0b813a840}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc076bb32f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804804 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc082aa92f0}, 0xc0420ad6e0, 0x1, 0x708017d6110a9a, 0x37e11d600, 0xc06b794f00, 0xc061a16de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc082aa92f0}, 0x51cdcd8, 0xc095cc7230, 0x95cc7138, 0x4c166a, 0x18, 0xc06b794f00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc082aa92f0}, 0xc06b794ea0, {{{{0x0, 0x0}, {0xc052450fc0, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc082aa92f0}, 0x6352410, 0x199f0468) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc082aa92f0}, {0xc06b794d80, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc082aa92f0}, 0xc01f12c3c0, 0xc095cc79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc082aa92f0}, 0x660089, {0xc06b794d80, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc082aa92f0}, 0xc0ae80c700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc082aa9200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082aa9200}, {0xc095cc93f8, 0x660089}, 0xc095cc9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc082aa9200}, {0xc0015221e0}, 0xc0ae80c500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc082aa9170}, 0xc0ae80c500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}, 0xc023048de0, 0xc0420ad6b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}, 0x3, 0xc00ecc6840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc095cc9980}}}, {0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}, 0xc023048de0, 0xc00ecc6840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc082aa9170}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082aa9170}, {0x40, 0xc095cc9b70}, 0xc095cc9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}, 0x203003, 0x203003) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc082aa9170}, {0x4e9d7a0, 0xc0ae80c500}, 0xc023048de0, 0xc0420ad6b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc082aa9170}, 0xc06c737b00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af93680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af93680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808218 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc03062b7a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28954 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run(0xc00d9916b0, {0x63c85b8, 0xc02a7fd0b0}, 0xc000d3ab40, {0x643def0, 0xc017cf0f10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:342 +0x1cb github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Receiver).PushUpdates(0xc01876a6c0, {0x643def0, 0xc017cf0f10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:89 +0x134 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb._SideTransport_PushUpdates_Handler({0x4cd47a0, 0xc01876a6c0}, {0x6400fe0, 0xc007765680}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:346 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}, 0xc02c48ce88, 0x51c46f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}, 0x3, 0xc00349cec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0xc00102e340}}}, {0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}, 0xc02c48ce88, 0xc00349cec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02a7fcff0}, {0x460ef4, 0xc01117dbc8}, 0xc01117db70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}, 0xc02c48ce88, 0xc00349cec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc014284b60}, 0xc02c48ce88, 0x51c46f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc01979ab60}, 0xc02c76e6c0, 0xc007b7b7a0, 0x9181540, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979ab60}, 0xc02c76e6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807293 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a1cc0, {0x63c85b8, 0xc046d8a900}, 0xc01d607d10, 0x1, 0xc0030170a8, 0x519bbc, 0xc084914240, 0xc065f2c780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc073120c60, {0x63c85b8, 0xc046d8a900}, 0xc073120c90, 0xc0030171a8, 0x30, 0x40, 0xffffffffffffffff, 0xc084914240, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a1cc0, {0x63c85b8, 0xc046d8a900}, 0xc084914000, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0774e4b40, 0x2}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046d8a900}, 0x63521a0, 0x6618bf80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a30280, {0x63c85b8, 0xc046d8a900}, {0xc033690a20, {0x1743010ffb232573, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a30280, {0x63c85b8, 0xc046d8a900}, 0xc04685cdc0, 0xc0030179f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046d8a900}, 0x660089, {0xc033690a20, {0x1743010ffb232573, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021a38400, {0x63c85b8, 0xc046d8a900}, 0xc09903ae00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffb232573, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffb232573, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046d8a870}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046d8a870}, {0xc0030193f8, 0x660089}, 0xc003019400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046d8a870}, {0xc0015221e0}, 0xc09903ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046d8a7b0}, 0xc09903ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}, 0xc01e64afa0, 0xc0603c2a20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}, 0x3, 0xc0619ff8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003019980}}}, {0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}, 0xc01e64afa0, 0xc0619ff8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046d8a7b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046d8a7b0}, {0x40, 0xc003019b70}, 0xc003019ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046d8a7b0}, {0x4e9d7a0, 0xc09903ac00}, 0xc01e64afa0, 0xc0603c2a20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046d8a7b0}, 0xc0534ae720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0783d0a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0783d0a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810094 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc01d33bd70}, 0xc059954800, 0xc0499d6460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01d33bd70}, 0xc059954800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c2be0ec, 0x0, 0x0}, 0xc03cc769f0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c2be0ec, 0x0, 0x0}, 0xc03cc769f0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01d33bce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d33bce0}, {0xc0179673f8, 0x660089}, 0xc017967400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01d33bce0}, {0x7fe3953db1f8}, 0xc059954500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01d33bc20}, 0xc059954500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}, 0xc00f3f4120, 0xc03135a258) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}, 0x3, 0xc063d6a700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017967980}}}, {0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}, 0xc00f3f4120, 0xc063d6a700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01d33bc20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d33bc20}, {0x40, 0xc017967b70}, 0xc017967ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01d33bc20}, {0x4e9d7a0, 0xc059954500}, 0xc00f3f4120, 0xc03135a258) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01d33bc20}, 0xc0aadfb920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc022bd5b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc022bd5b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808037 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc06b5c7740}, 0xc04aa6e5d0, 0x1, 0xc0057a90a8, 0x519bbc, 0xc073921500, 0xc08e32d320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06b5c7740}, 0x1e0, 0xc0057a9230, 0x739214a0, 0x0, 0x660069, 0xc073921500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc06b5c7740}, 0xc0739214a0, {{{{0x0, 0x0}, {0xc06040ad80, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06b5c7740}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc06b5c7740}, {0xc073d845a0, {0x174301105988909c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc06b5c7740}, 0xc02d18db80, 0xc0057a99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b5c7740}, 0x660089, {0xc073d845a0, {0x174301105988909c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc06b5c7740}, 0xc04cd21b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105988909c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105988909c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b5c76b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b5c76b0}, {0xc0057ab3f8, 0x660089}, 0xc0057ab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b5c76b0}, {0xc0015221e0}, 0xc04cd21900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b5c7620}, 0xc04cd21900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}, 0xc03d580d80, 0xc01f06dba8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}, 0x3, 0xc0762ec1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0057ab980}}}, {0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}, 0xc03d580d80, 0xc0762ec1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b5c7620}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b5c7620}, {0x40, 0xc0057abb70}, 0xc0057abab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b5c7620}, {0x4e9d7a0, 0xc04cd21900}, 0xc03d580d80, 0xc01f06dba8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b5c7620}, 0xc040d73620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0749c66c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0749c66c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812421 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc06b7bae70}, 0xc03c1eaa50, 0x1, 0xc019c630a8, 0x519bbc, 0xc04c6d9360, 0xc0d6109080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0341c17a0, {0x63c85b8, 0xc06b7bae70}, 0xc0341c17e0, 0xc019c631a8, 0x4bd14e, 0x48d7e7, 0xc019c63130, 0xc04c6d9360, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc06b7bae70}, 0xc03d5857a0, {{{{0xc073da7680, 0x5}, {0xc012539560, 0xc}}, {{0xc0355166c0, 0x18}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06b7bae70}, 0x63521a0, 0x204d2c18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc06b7bae70}, {0xc0440ee000, {0x17430110145c5f39, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc06b7bae70}, 0xc04d955400, 0xc000000000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b7bae70}, 0x660089, {0xc0440ee000, {0x17430110145c5f39, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc06b7bae70}, 0xc070374f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110145c5f39, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110145c5f39, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b7bade0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b7bade0}, {0xc019c653f8, 0x660089}, 0xc019c65400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b7bade0}, {0x0}, 0xc070374900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b7bad20}, 0xc070374900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}, 0xc05be86ea0, 0xc0b9d71818) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}, 0x3, 0xc060ceb500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc019c65980}}}, {0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}, 0xc05be86ea0, 0xc060ceb500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b7bad20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b7bad20}, {0x40, 0xc019c65b70}, 0xc019c65ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b7bad20}, {0x4e9d7a0, 0xc070374900}, 0xc05be86ea0, 0xc0b9d71818) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b7bad20}, 0xc03d585620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0a2a7fe60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0a2a7fe60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807807 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc0458d1890}, 0xc03f8d66f0, 0x1, 0x0, 0x1, 0xc079f19800, 0xc07789acc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0458d1890}, 0x120, 0xc00f266728, 0x79f197a0, 0x0, 0x660069, 0xc079f19800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc0458d1890}, 0xc079f197a0, {{{{0x0, 0x0}, {0xc0ada1b9e0, 0x14}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0458d1890}, 0x6352410, 0x2ad6f700) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc0458d1890}, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc0458d1890}, 0xc042f32320, 0xc089d54ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0458d1890}, 0x660089, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc0458d1890}, 0xc0625cf300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0458d17a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0458d17a0}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0458d17a0}, {0xc00102e4e0}, 0xc0625cf200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0458d1740}, 0xc0625cf200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0373225d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 202012874 [select, 31 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc040bc63e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc059560ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}, 0xc025b512d8, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}, 0x3, 0xc00f06d640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0xc02aadef90}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}, 0xc025b512d8, 0xc00f06d640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027c44c60}, {0x460ef4, 0xc006594bc8}, 0xc006594b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}, 0xc025b512d8, 0xc00f06d640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc036234c30}, 0xc025b512d8, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01aec0120, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01aec0120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219787509 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0493a6b50, 0x1c) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0be1c1500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0be1c1b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0e0dd2600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0e0dd2600, {0x63c8510, 0xc0493a6b80}, 0xc04c2f5dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04b4bd500) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0493a6b80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04b4bd500) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 28390 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe994b8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc021538000, 0xc00db28900, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc021538000, {0xc00db28900, 0x811, 0x811}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc021538000, {0xc00db28900, 0x773, 0xc031d323c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01978adb8, {0xc00db28900, 0x7eda79, 0xc00db289a3}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc012fd3260, {0xc00db28900, 0x0, 0x811}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0490966c0, {0xc00db28900, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0022decf8, {0x631b900, 0xc0490966c0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0022dea80, {0x7fe42ab59310, 0xc012fd3260}, 0x773) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0022dea80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0022dea80, {0xc004626000, 0x8000, 0x840010601}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc014825620, {0xc002260740, 0x9, 0x18}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc014825620}, {0xc002260740, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc002260740, 0x9, 0xc08e777560}, {0x631b600, 0xc014825620}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc002260700) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002944b60, 0x10bed2f, 0x63c85b8) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc002944b60}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 219773314 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafee9b50, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07ae18a00, 0xc016b4a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07ae18a00, {0xc016b4a000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07ae18a00, {0xc016b4a000, 0x735, 0xc07ad55cc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0650dc638, {0xc016b4a000, 0x7eda79, 0xc016b4a005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc012f470e0, {0xc016b4a000, 0xc083b68c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02902dbd8, {0xc016b4a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0712d7e78, {0x631b900, 0xc02902dbd8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0712d7c00, {0x7fe42ab59310, 0xc012f470e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0712d7c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0712d7c00, {0xc0aac8e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01576ff98, {0xc0aac8e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06e306b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06e306b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06e3070e0, {0x634aac0, 0xc06e306b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06e306a80, 0xc041516790, 0xc083b6942b, 0xc083b6942d, {0x635bec0, 0xc06485cfc0}, 0xc000d6ae00, 0xc01abee4b4, 0xc083b69429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06e306a80, {0x63c85b8, 0xc083611ec0}, 0xc041516780, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc078a02038, 0x8}}, 0x0, 0xc083611e60, 0xc083611e90, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc083611ec0}, {0x643d810, 0xc012f470e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc055b1cea0}, {0x643d810, 0xc012f470e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc055b1cea0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800449 [semacquire]: sync.runtime_SemacquireMutex(0xc001fc1040, 0x3, 0xc001fc1040) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074b2c0) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc095b3c338) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074b2b0, 0x11869e5, 0x7fe1a3f69800, 0x7fa3, 0x7fe23ec364c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc02551fc80, 0x7fe1a3f69801, 0x7fe1a3f69820, 0x4038, 0x403d) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0254bcb00, {0x5cd88ed, 0x4038}, 0x0, 0xc056405d78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc056405b00, {0xffffffffffffff15, 0xc095b3c5c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc056405b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc056405b00, {0xc036380e70, 0x2b, 0x6d}, 0x13, 0x27) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc056405b00, {0xc021ee8500, 0x2b, 0x6d}, {0xc036380e70, 0x51c9860, 0xc036380e70}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc056405b00, {0xc021ee8500, 0x2b, 0x75}, {0xc036380e70, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b2314398, {0xc021ee8500, 0x2b, 0x75}, {0xc036380e70, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b2312cc0, {0xc036380e70, 0x6d, 0x75}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b2312cc0, {0xc021ee8500, 0x2b, 0xc036380e70}, {0xc036380e70, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b2312a00, {0xc036380e70, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0616be008, {{0xc03dbd7ad0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc082920c00, {{0xc03dbd7ad0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0590b1c00, {0x63c85b8, 0xc0c74cf590}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0c74cf590}, {0x64a6560, 0xc082920c00}, {0xc03dbd7ad0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0c74cf590}, {0x6466ed8, 0xc0616be000}, {0xc03dbd7ad0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01b5ef6b0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc071b6fd00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b0ec00, {0x63c85b8, 0xc0c74cf590}, 0xc071b6fd00, 0xc066747720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b0ec00, {0x63c85b8, 0xc0c74cf590}, 0xc071b6fd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c74cf500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c74cf500}, {0xc0ad8388f0, 0x660089}, 0xc0ad8388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c74cf500}, {0xc00102e4e0}, 0xc071b6fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c74cf4a0}, 0xc071b6fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06b4b8bd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28061 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc0122d8800, 0x6437410, 0xc0178b39e0, {0x7fe42b743500, 0xc007d05220}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:254 +0x845 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:150 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc0082db200, 0xc0178b39e0, {0x6437410, 0xc007d05220}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3082 +0xf7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc001fc11e0, 0xc0178b39e0, {0x6437410, 0xc007d05220}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:216 +0xf6 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x6437410, 0x63c85b8, {0x6437410, 0xc007d05220}) github.com/cockroachdb/cockroach/pkg/server/node.go:1227 +0x38 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:753 +0xeb created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:750 +0x1cf goroutine 219806482 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc07d8bc420}, 0xc0909dc0a8, 0x1, 0x0, 0x1, 0xc072299020, 0xc0741ccba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07d8bc420}, 0x120, 0xc0061a0728, 0x72298fc0, 0x0, 0x660069, 0xc072299020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc07d8bc420}, 0xc072298fc0, {{{{0x0, 0x0}, {0xc052ccc480, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07d8bc420}, 0x6352410, 0x2016de38) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc07d8bc420}, {0xc072298b40, {0x174301104dbcbff6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc07d8bc420}, 0xc075b4fc20, 0xc07685aee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07d8bc420}, 0x660089, {0xc072298b40, {0x174301104dbcbff6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc07d8bc420}, 0xc048472a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104dbcbff6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104dbcbff6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07d8bc390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07d8bc390}, {0xc0935988f0, 0x660089}, 0xc0935988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07d8bc390}, {0xc00102e4e0}, 0xc048472900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07d8bc330}, 0xc048472900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07c7c3fb0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28123 [select]: github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*jobMonitor).start.func1({0x63c85b8, 0xc02ca0c090}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/scheduled_job_monitor.go:96 +0x267 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812588 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812650 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812034 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024f05a40, {0x63c85b8, 0xc0287987e0}, 0xc0ce7a4078, 0x1, 0x0, 0x1, 0xc085b9fda0, 0xc018ab24e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0287987e0}, 0x120, 0xc03600a728, 0x85b9fd40, 0x0, 0x660069, 0xc085b9fda0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024f05a40, {0x63c85b8, 0xc0287987e0}, 0xc085b9fd40, {{{{0x0, 0x0}, {0xc0bbf430e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0287987e0}, 0x6352410, 0x204d2388) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024ef0fa0, {0x63c85b8, 0xc0287987e0}, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024ef0fa0, {0x63c85b8, 0xc0287987e0}, 0xc01c050c80, 0xc07c880ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0287987e0}, 0x660089, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024f6b000, {0x63c85b8, 0xc0287987e0}, 0xc060fbb900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0287986c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0287986c0}, {0xc0ad8368f0, 0x660089}, 0xc0ad8368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0287986c0}, {0xc00102e4e0}, 0xc060fbb800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc028798660}, 0xc060fbb800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01f1b7380}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802780 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01df61400, {0x63c85b8, 0xc0466e6cf0}, 0xc0ab4fe100, 0xc024142500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01df61400, {0x63c85b8, 0xc0466e6cf0}, 0xc0ab4fe100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0466e6c60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0466e6c60}, {0xc06b8553f8, 0x660089}, 0xc06b855400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0466e6c60}, {0x7fe3e2122a70}, 0xc048573e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0466e6bd0}, 0xc048573e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}, 0xc05b8f3380, 0xc0170cb560) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}, 0x3, 0xc01e65d7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06b855980}}}, {0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}, 0xc05b8f3380, 0xc01e65d7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0466e6bd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0466e6bd0}, {0x40, 0xc06b855b70}, 0xc06b855ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0466e6bd0}, {0x4e9d7a0, 0xc048573e00}, 0xc05b8f3380, 0xc0170cb560) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0466e6bd0}, 0xc065eb6ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0ba6445a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0ba6445a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806175 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc078c759b0}, 0xc06d3adc00, 0xc0c59ae320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc078c759b0}, 0xc06d3adc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d51d23d, 0x0, 0x0}, 0xc07b9f8ea0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d51d23d, 0x0, 0x0}, 0xc07b9f8ea0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078c75920}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078c75920}, {0xc00b40d3f8, 0x660089}, 0xc00b40d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078c75920}, {0x7265765f77736d72}, 0xc06d3ad900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078c75890}, 0xc06d3ad900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}, 0xc07947bec0, 0xc08cc8edc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}, 0x3, 0xc05ac11280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b40d980}}}, {0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}, 0xc07947bec0, 0xc05ac11280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc078c75890}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078c75890}, {0x40, 0xc00b40db70}, 0xc00b40dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc078c75890}, {0x4e9d7a0, 0xc06d3ad900}, 0xc07947bec0, 0xc08cc8edc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc078c75890}, 0xc03abd4060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc091a5ac60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc091a5ac60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812110 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc04a42ca20}, 0xc0387f9cb0, {0x646cc48, 0xc04a42ca80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc04a42ca20}, 0xedb7a8350, 0xc060ab9e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc04a42ca20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06b5f4990}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0387f9ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06b5f4990}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812593 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219789414 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc04d8af200}, 0xc01526ecb0, {0x646cc48, 0xc04d8af260}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc04d8af200}, 0xedb7a834d, 0xc0384a65a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc04d8af200}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03c1042a0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01526eef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc03c1042a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806549 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e83a800, {0x63c85b8, 0xc08aa08420}, 0xc062701300, 0xc075bfc640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e83a800, {0x63c85b8, 0xc08aa08420}, 0xc062701300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104f1d9ac1, 0x0, 0x0}, 0xc089fa43b0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104f1d9ac1, 0x0, 0x0}, 0xc089fa43b0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08aa08390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08aa08390}, {0xc08da97918, 0x660089}, 0xc08da97920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08aa08390}, {0xc00102e4e0}, 0xc062701200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08aa08330}, 0xc062701200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc065b99a40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc08aa082a0}, {0xc00dfd36b0, 0x24}, 0x46c7a15a4, 0xc001f12d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc03469eba0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc08aa082a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28485 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc005e42180, {0x63c85b8, 0xc01cbda840}, 0x73dd, 0xc011c01d88, 0xc000d3ab40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:269 +0x53f github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x63c85b8, 0xc01cbda840}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:222 +0x3c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28588 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219788945 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0242a3980}, 0xc024605cb0, {0x646cc48, 0xc0242a39e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0242a3980}, 0xedb7a834d, 0xc08ac1cfc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0242a3980}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc023613590}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc024605ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc023613590}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 517826 [select, 8 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc.(*Allocator).start.func1({0x63c85b8, 0xc03c1a9d10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc/id_alloc.go:151 +0x735 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 214159444 [sync.Cond.Wait, 9 minutes]: sync.runtime_notifyListWait(0xc057742550, 0xa) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc08d28e000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc08d28e660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04633f300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04633f300, {0x63c8510, 0xc057742580}, 0xc05164fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0978fe570) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc057742580}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0978fe570) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812723 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x80, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0532f9880, 0xc03dd83158) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0532f9880, {0xc0da1d8930, 0x2e, 0x6d}, {0xc017870900, 0x2e, 0x72}, 0x0, 0x51) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc093395088, {0xc0da1d8930, 0x1, 0x72}, {0xc017870900, 0x56, 0xc03dd83298}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093395088, {0xc0da1d8930, 0x2e, 0x6d}, {0xc017870900, 0x2e, 0x72}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0933947c0, {0xc017870900, 0x2, 0xc}, 0x2b, 0xc8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0933947c0, {0xc0da1d8930, 0x72, 0xc050c9afd0}, {0xc017870900, 0xc050c9afd0, 0x2d}, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093394500, {0xc017870900, 0x2e, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc051d5b328, {{0xc050c9afd0, 0x2d, 0xb0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc055b25200, {{0xc051de18c0, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08c160c00, {0x63c85b8, 0xc062cde2a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062cde2a0}, {0x64a6560, 0xc055b25200}, {0xc051de18c0, 0xc00bc1fa40, 0x30}, {0x1743011093209cde, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062cde2a0}, {0x6466ed8, 0xc051d5b080}, {0xc051de18c0, 0x26, 0x30}, {0x1743011093209cde, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01675e528}, {{0x1743011093209cde, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011093209cde, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc015101500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b0ee01de, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc016126000, {0x63c85b8, 0xc062cde2a0}, 0xc015101500, 0xc04bf3adc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016126000, {0x63c85b8, 0xc062cde2a0}, 0xc015101500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011093209cde, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011093209cde, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cde210}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cde210}, {0xc03dd893f8, 0x660089}, 0xc03dd89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cde210}, {0x10090000900f}, 0xc015101200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cde150}, 0xc015101200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}, 0xc06ac129c0, 0xc041139ec0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}, 0x3, 0xc01c0bd780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03dd89980}}}, {0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}, 0xc06ac129c0, 0xc01c0bd780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cde150}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cde150}, {0x40, 0xc03dd89b70}, 0xc03dd89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cde150}, {0x4e9d7a0, 0xc015101200}, 0xc06ac129c0, 0xc041139ec0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cde150}, 0xc049257200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a5320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a5320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 17537 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 219809816 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc087449bf0}, 0xc01227aa80, 0x1, 0x0, 0x1, 0xc0c9892960, 0xc0603d8a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc087449bf0}, 0x120, 0xc019ec4728, 0xc9892900, 0x0, 0x660069, 0xc0c9892960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc087449bf0}, 0xc0c9892900, {{{{0x0, 0x0}, {0xc0438bd8c0, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc087449bf0}, 0x6352410, 0x29b7b7e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc087449bf0}, {0xc085ab8d80, {0x1743011066dd663a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc087449bf0}, 0xc0743e34a0, 0xc00b2c0ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc087449bf0}, 0x660089, {0xc085ab8d80, {0x1743011066dd663a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc087449bf0}, 0xc082f58700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066dd663a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066dd663a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc087449b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc087449b60}, {0xc002fe68f0, 0x660089}, 0xc002fe68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc087449b60}, {0xc00102e4e0}, 0xc082f58600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc087449b00}, 0xc082f58600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc034532e40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811791 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc06e85d250, {0x63c85b8, 0xc028799260}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc028799260}, {0x4f67ad2, 0x10beec0}, 0xc0387f9fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 28210 [select, 48 minutes]: github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc006bb02d0, {0x63c85b8, 0xc02c253950}) github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:354 +0x176 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28228 [select, 10 minutes]: github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9({0x63c85b8, 0xc02ca0c8d0}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:839 +0x196 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803672 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc074591a40}, 0xc028879b90, 0x1, 0x0, 0x1, 0xc03c285ec0, 0xc04a85c840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc074591a40}, 0x120, 0xc0767d4728, 0x3c285e60, 0x0, 0x660069, 0xc03c285ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc074591a40}, 0xc03c285e60, {{{{0x0, 0x0}, {0xc040438a20, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc074591a40}, 0x6352410, 0x34330598) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc074591a40}, {0xc03b940c60, {0x174301100d71b895, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc074591a40}, 0xc06fbc8aa0, 0xc076856ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc074591a40}, 0x660089, {0xc03b940c60, {0x174301100d71b895, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc074591a40}, 0xc0d503ed00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100d71b895, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100d71b895, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074591980}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074591980}, {0xc01fad48f0, 0x660089}, 0xc01fad48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074591980}, {0xc00102e4e0}, 0xc0d503ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074591920}, 0xc0d503ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc037feb620}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811383 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021cd2000, {0x63c85b8, 0xc092ec50e0}, 0xc06613d590, 0x1, 0xc02e3250a8, 0x519bbc, 0xc06dcb7aa0, 0xc06ee17740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc046250fc0, {0x63c85b8, 0xc092ec50e0}, 0xc046251010, 0xc02e3251a8, 0x30, 0x40, 0xffffffffffffffff, 0xc06dcb7aa0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021cd2000, {0x63c85b8, 0xc092ec50e0}, 0xc06dcb79e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc01542eea0, 0xe}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc092ec50e0}, 0x6352260, 0x202c4d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c432c0, {0x63c85b8, 0xc092ec50e0}, {0xc06dcb78c0, {0x174301103b05a390, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c432c0, {0x63c85b8, 0xc092ec50e0}, 0xc073035f40, 0xc02e3259f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc092ec50e0}, 0x660089, {0xc06dcb78c0, {0x174301103b05a390, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd6000, {0x63c85b8, 0xc092ec50e0}, 0xc01ac99f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b05a390, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b05a390, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc092ec5020}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092ec5020}, {0xc02e3273f8, 0x660089}, 0xc02e327400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc092ec5020}, {0xc01c6b8400}, 0xc01ac99d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc092ec4f90}, 0xc01ac99d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}, 0xc05fc32c00, 0xc06613d6b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}, 0x3, 0xc06c927300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02e327980}}}, {0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}, 0xc05fc32c00, 0xc06c927300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc092ec4f90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092ec4f90}, {0x40, 0xc02e327b70}, 0xc02e327ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc092ec4f90}, {0x4e9d7a0, 0xc01ac99d00}, 0xc05fc32c00, 0xc06613d6b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc092ec4f90}, 0xc03d4991a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06dcb77a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06dcb77a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812616 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc03bd10a80, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc0423c7198, 0xc0b1ed7a38, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc0b1ed7b50, 0x0, 0xc0b1ed7b50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0b1ed79e8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b1ed79e8, {0xc0bf865200, 0x18, 0x60}, {0xc02d5ae460, 0x18, 0x4b}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b1ed67c0, {0xc02d5ae460, 0x0, 0xc0423c74e0}, 0x121ea30, 0x50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b1ed67c0, {0xc0bf865200, 0x18, 0x4b}, {0xc02d5ae460, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b1ed6500, {0xc02d5ae460, 0x18, 0x4b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc051d5cc08, {{0xc08467aa08, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc059f4d900, {{0xc08467aa08, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04e059400, {0x63c85b8, 0xc0304caa80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0304caa80}, {0x64a6560, 0xc059f4d900}, {0xc08467aa08, 0x0, 0x18}, {0x1743011095a97215, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0304caa80}, {0x6466ed8, 0xc051d5cc00}, {0xc08467aa08, 0x17, 0x18}, {0x1743011095a97215, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc064ac06a8}, {{0x1743011095a97215, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011095a97215, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0460ded00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b376d715, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc0304caa80}, 0xc0460ded00, 0xc04e4cdcc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc0304caa80}, 0xc0460ded00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011095a97215, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011095a97215, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0304ca990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304ca990}, {0xc0423cd3f8, 0x660089}, 0xc0423cd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0304ca990}, {0x408c37eb1b156869}, 0xc0460deb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0304ca900}, 0xc0460deb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}, 0xc03ebdd7e0, 0xc031cfb3e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}, 0x3, 0xc015b9e300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0423cd980}}}, {0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}, 0xc03ebdd7e0, 0xc015b9e300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0304ca900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304ca900}, {0x40, 0xc0423cdb70}, 0xc0423cdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0304ca900}, {0x4e9d7a0, 0xc0460deb00}, 0xc03ebdd7e0, 0xc031cfb3e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0304ca900}, 0xc039f0b920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794611 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc081c94548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc081c945b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07c5fe5f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0caa94380, {0xc0536cf6c0, 0x2b, 0x6d}, {0xc06b752930, 0x2b, 0x6b}, 0x1, 0xe5) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07c5fe488, {0xc0536cf6c0, 0x1, 0x6b}, {0xc06b752930, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07c5fe488, {0xc0536cf6c0, 0x2b, 0x6d}, {0xc06b752930, 0x2b, 0x6b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07c5fdbc0, {0xc06b752930, 0x6b, 0x6d}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07c5fdbc0, {0xc0536cf6c0, 0x2b, 0xc06b752930}, {0xc06b752930, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07c5fd900, {0xc06b752930, 0x2b, 0x6b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06c4e6b08, {{0xc051eb6ea0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc078923600, {{0xc051eb6ea0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc01ecb9800, {0x63c85b8, 0xc056522d20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056522d20}, {0x64a6560, 0xc078923600}, {0xc051eb6ea0, 0x70, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056522d20}, {0x6466ed8, 0xc06c4e6b00}, {0xc051eb6ea0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c77f4e8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc065b86800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020037c00, {0x63c85b8, 0xc056522d20}, 0xc065b86800, 0xc048adfe00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020037c00, {0x63c85b8, 0xc056522d20}, 0xc065b86800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056522c90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056522c90}, {0xc09365a8f0, 0x660089}, 0xc09365a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056522c90}, {0xc00102e4e0}, 0xc065b86600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056522bd0}, 0xc065b86600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc073172900}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 214159411 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af29bb8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc079b97b80, 0xc00c2eb000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc079b97b80, {0xc00c2eb000, 0x73d, 0x73d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc079b97b80, {0xc00c2eb000, 0x0, 0xc058e2cbc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06541e520, {0xc00c2eb000, 0x40, 0x7f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06d20a180, {0xc00c2eb000, 0xc0235e0c60, 0x73d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03d4af518, {0xc00c2eb000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc051357078, {0x631b900, 0xc03d4af518}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc051356e00, {0x7fe42ab59310, 0xc06d20a180}, 0x73d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc051356e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc051356e00, {0xc0c90ac000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0b76a2258, {0xc0c90ac000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc08d28e088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc08d28e088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc08d28e660, {0x634aac0, 0xc08d28e088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc08d28e000, 0xc0978fe550, 0xc058e2d42b, 0xc058e2d42d, {0x635bec0, 0xc07415ef30}, 0xc000d6ae00, 0xc0192df604, 0xc058e2d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc08d28e000, {0x63c85b8, 0xc01efe5f50}, 0xc0978fe540, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09a6fc033, 0x10}}, 0x0, 0xc01efe5ef0, 0xc01efe5f20, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc01efe5f50}, {0x643d810, 0xc06d20a180}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0755514a0}, {0x643d810, 0xc06d20a180}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0755514a0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28286 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc0140e1cf8) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01be138f0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01be138f0, {0x63c8510, 0xc017d65400}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00167cf00, {0x63c8510, 0xc017d653c0}, {0xc02cafa0f0, 0x1, 0xc001b02f28}, {0x1cf6ee7, 0xd3ab40, 0xc0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:139 +0x625 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc001b02f10, {0x63c8510, 0xc017d653c0}, {0xc02cafa0f0, 0xc001b02f98, 0x1893c7b}, {0xf0f00c00d513300, 0x63c85b8, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:78 +0x45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x63c8510, 0xc017d653c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:285 +0x7a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219809845 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0519e09f0}, 0xc03f181920, 0x1, 0xc0282030a8, 0x519bbc, 0xc0735e9aa0, 0xc05f848240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0519e09f0}, 0x1e0, 0xc028203230, 0x735e9a40, 0x0, 0x660069, 0xc0735e9aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0519e09f0}, 0xc0735e9a40, {{{{0x0, 0x0}, {0xc028414900, 0x5e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0519e09f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0519e09f0}, {0xc0744d9b00, {0x174301105ee9eb70, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0519e09f0}, 0xc070dc0280, 0xc0282039f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0519e09f0}, 0x660089, {0xc0744d9b00, {0x174301105ee9eb70, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0519e09f0}, 0xc02927ca00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105ee9eb70, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105ee9eb70, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0519e0960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0519e0960}, {0xc0282053f8, 0x660089}, 0xc028205400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0519e0960}, {0x506f635346567675}, 0xc02927c800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0519e08d0}, 0xc02927c800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}, 0xc03bcc60c0, 0xc02f3dd248) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}, 0x3, 0xc0426c3a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028205980}}}, {0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}, 0xc03bcc60c0, 0xc0426c3a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0519e08d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0519e08d0}, {0x40, 0xc028205b70}, 0xc028205ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0519e08d0}, {0x4e9d7a0, 0xc02927c800}, 0xc03bcc60c0, 0xc02f3dd248) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0519e08d0}, 0xc0a3016ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0744d99e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0744d99e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811823 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc05327f8f0}, 0xc0212405a0, 0x1, 0xc032d1bff0, 0x519bbc, 0xc02895c060, 0xc073d61500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc05327f8f0}, 0x1e0, 0xc032d1c178, 0x2895c000, 0x0, 0x660069, 0xc02895c060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc05327f8f0}, 0xc02895c000, {{{{0x0, 0x0}, {0xc06c58ad80, 0x6a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc05327f8f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc05327f8f0}, {0xc04305de60, {0x174301107bade4d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc05327f8f0}, 0xc068976aa0, 0xc058160938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05327f8f0}, 0x660089, {0xc04305de60, {0x174301107bade4d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc05327f8f0}, 0xc04c24a600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107bade4d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107bade4d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05327f830}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05327f830}, {0xc058162340, 0x660089}, 0xc058162348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05327f830}, {0xc00102e4e0}, 0xc04c24a500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05327f7d0}, 0xc04c24a500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc042c1d6b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28227 [select]: github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8({0x63c85b8, 0xc02ca0c8a0}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:807 +0x19f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812644 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812640 [runnable]: math/big.nat.montgomery({0xc070894900, 0x10, 0x24}, {0xc001a0a000, 0x10, 0x26}, {0xc001a0a000, 0x10, 0x26}, {0xc000b9bc30, ...}, ...) GOROOT/src/math/big/nat.go:213 +0x4ef math/big.nat.expNNMontgomery({0xc070894900, 0x25, 0x24}, {0xc0b15a26c0, 0x20, 0x44}, {0xc000c52dc0, 0x10, 0x20}, {0xc000b9bc30, ...}) GOROOT/src/math/big/nat.go:1123 +0xb8a math/big.nat.expNN({0x0, 0x460ef4, 0x7fe457d7f5b8}, {0xc0b15a26c0, 0xc00310ac08, 0xc0b15a26c0}, {0xc000c52dc0, 0x7fe3b7ada600, 0x10}, {0xc000b9bc30, ...}) GOROOT/src/math/big/nat.go:937 +0x432 math/big.(*Int).Exp(0xc07c608be8, 0xc07c608c08, 0xc00310ac28, 0xcae7f876c222f92a) GOROOT/src/math/big/int.go:509 +0x199 crypto/rsa.decrypt({0x631b8c0, 0xc00011afc0}, 0xc000f3fb60, 0xc00310ad58) GOROOT/src/crypto/rsa/rsa.go:536 +0x495 crypto/rsa.decryptAndCheck({0x631b8c0, 0xc00011afc0}, 0xc000f3fb60, 0xc04f1ab700) GOROOT/src/crypto/rsa/rsa.go:570 +0x31 crypto/rsa.signPSSWithSalt({0x631b8c0, 0xc00011afc0}, 0xc000f3fb60, 0x0, {0xc04921baa0, 0x20, 0x20}, {0xc04921bac0, 0x20, 0x20}) GOROOT/src/crypto/rsa/pss.go:217 +0x148 crypto/rsa.SignPSS({0x631b8c0, 0xc00011afc0}, 0xc000f3fb60, 0x0, {0xc04921baa0, 0x20, 0x20}, 0x4b30720) GOROOT/src/crypto/rsa/pss.go:281 +0x1f5 crypto/rsa.(*PrivateKey).Sign(0xc02a2dbc00, {0x631b8c0, 0xc00011afc0}, {0xc04921baa0, 0xc02a2dbc00, 0x6}, {0x631b8e0, 0xc03b489f10}) GOROOT/src/crypto/rsa/rsa.go:146 +0x66 crypto/tls.(*serverHandshakeStateTLS13).sendServerCertificate(0xc07c609148) GOROOT/src/crypto/tls/handshake_server_tls13.go:623 +0x42b crypto/tls.(*serverHandshakeStateTLS13).handshake(0xc07c609148) GOROOT/src/crypto/tls/handshake_server_tls13.go:62 +0x85 crypto/tls.(*Conn).serverHandshake(0xc0787b2a80, {0x63c8510, 0xc050a073c0}) GOROOT/src/crypto/tls/handshake_server.go:54 +0xd0 crypto/tls.(*Conn).handshakeContext(0xc0787b2a80, {0x63c8548, 0xc000140068}) GOROOT/src/crypto/tls/conn.go:1453 +0x36f crypto/tls.(*Conn).HandshakeContext(...) GOROOT/src/crypto/tls/conn.go:1396 crypto/tls.(*Conn).Handshake(...) GOROOT/src/crypto/tls/conn.go:1380 crypto/tls.(*Conn).Read(0xc0787b2a80, {0xc073d14c48, 0x4, 0xc019549520}) GOROOT/src/crypto/tls/conn.go:1265 +0x65 io.ReadAtLeast({0x7fe456a34450, 0xc0787b2a80}, {0xc073d14c48, 0x4, 0x4}, 0x4) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadUntypedMsg(0xc073d14c30, {0x7fe456a34450, 0xc0787b2a80}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:151 +0x65 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).readVersion(0xc00c689e00, {0x7fe456a34450, 0xc0787b2a80}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1357 +0x11f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).maybeUpgradeToSecureConn(0xc00c689e00, {0x0, 0xffc}, {0x643d810, 0xc04d73b0e0}, 0x2, 0x4d2162f, 0xc07c609be0) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1301 +0x367 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c8510, 0xc050a07380}, {0x643d810, 0xc04d73b0e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:807 +0x5a9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc073d14b70}, {0x643d810, 0xc04d73b0e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc073d14b70}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28064 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc006cb8460, {0x63c8510, 0xc00abaa7c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:310 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc006cb8460, {0x63c85b8, 0xc02c8fbb00}, 0xc02c134360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:335 +0xbb github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x63c85b8, 0xc02c8fbb00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806948 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021ad2c80, {0x63c85b8, 0xc01e001920}, 0xc01d9b60a8, 0x1, 0xc0038e10a8, 0x519bbc, 0xc048182ae0, 0xc04c05f980) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc03caee000, {0x63c85b8, 0xc01e001920}, 0xc048182ae0, 0xc0038e11a8, 0x30, 0x40, 0x5, 0xc048182ae0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021ad2c80, {0x63c85b8, 0xc01e001920}, 0xc048182a20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc042915200, 0xb}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01e001920}, 0x6352260, 0x3b956168) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a319a0, {0x63c85b8, 0xc01e001920}, {0xc048182900, {0x1743011016897f20, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a319a0, {0x63c85b8, 0xc01e001920}, 0xc072c5a500, 0xc0038e19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01e001920}, 0x660089, {0xc048182900, {0x1743011016897f20, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c06000, {0x63c85b8, 0xc01e001920}, 0xc0797e4400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011016897f20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011016897f20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e001650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e001650}, {0xc0038e33f8, 0x660089}, 0xc0038e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e001650}, {0x7fe4292d4e80}, 0xc0797e4200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e0015c0}, 0xc0797e4200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}, 0xc08ecfb080, 0xc030e760d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}, 0x3, 0xc090ff1fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0038e3980}}}, {0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}, 0xc08ecfb080, 0xc090ff1fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01e0015c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e0015c0}, {0x40, 0xc0038e3b70}, 0xc0038e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01e0015c0}, {0x4e9d7a0, 0xc0797e4200}, 0xc08ecfb080, 0xc030e760d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01e0015c0}, 0xc06e346120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0481827e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0481827e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811918 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021c5aa00, {0x63c85b8, 0xc03bea59e0}, 0xc085be1530, 0x1, 0xc00a7610a8, 0x519bbc, 0xc0bea70940, 0xc07abd31a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc023018480, {0x63c85b8, 0xc03bea59e0}, 0xc0bea70940, 0xc00a7611a8, 0x30, 0x40, 0x0, 0xc0bea70940, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021c5aa00, {0x63c85b8, 0xc03bea59e0}, 0xc0bea70700, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0154f7440, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03bea59e0}, 0x63521a0, 0x3c90c4e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c42820, {0x63c85b8, 0xc03bea59e0}, {0xc0708b4120, {0x174301102b3f69d0, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c42820, {0x63c85b8, 0xc03bea59e0}, 0xc0413c6a00, 0xc00a7619f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03bea59e0}, 0x660089, {0xc0708b4120, {0x174301102b3f69d0, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c6d400, {0x63c85b8, 0xc03bea59e0}, 0xc031f84200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b3f69d0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b3f69d0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03bea5950}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bea5950}, {0xc00a7633f8, 0x660089}, 0xc00a763400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03bea5950}, {0x7fe4286a88b8}, 0xc01f6fff00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03bea5860}, 0xc01f6fff00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}, 0xc03d9ac7c0, 0xc023414d68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}, 0x3, 0xc035ac7f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a763980}}}, {0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}, 0xc03d9ac7c0, 0xc035ac7f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03bea5860}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bea5860}, {0x40, 0xc00a763b70}, 0xc00a763ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03bea5860}, {0x4e9d7a0, 0xc01f6fff00}, 0xc03d9ac7c0, 0xc023414d68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03bea5860}, 0xc048f9efc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0708b4000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0708b4000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807334 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc02de4e2a0}, 0xc01e068990, 0x1, 0xc0042910a8, 0x519bbc, 0xc092ce9e40, 0xc05b8fcd20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc060492360, {0x63c85b8, 0xc02de4e2a0}, 0xc092ce9e40, 0xc0042911a8, 0x41, 0x60, 0x9, 0xc092ce9e40, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc02de4e2a0}, 0xc092ce9c00, {{{{0x0, 0x0}, {0xc04b0b2b40, 0x3}}, {{0xc07ca006c0, 0x9}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02de4e2a0}, 0x63521a0, 0x25ab6d48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc02de4e2a0}, {0xc07c42e240, {0x17430110069245bb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc02de4e2a0}, 0xc05a957400, 0xc0042919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02de4e2a0}, 0x660089, {0xc07c42e240, {0x17430110069245bb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc02de4e2a0}, 0xc08e4f3800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110069245bb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110069245bb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02de4e030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02de4e030}, {0xc0042933f8, 0x660089}, 0xc004293400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02de4e030}, {0x40e8faff2ddd4c3a}, 0xc08e4f3500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022369d70}, 0xc08e4f3500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}, 0xc088ac00c0, 0xc082101c68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}, 0x3, 0xc075ce3680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004293980}}}, {0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}, 0xc088ac00c0, 0xc075ce3680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022369d70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022369d70}, {0x40, 0xc004293b70}, 0xc004293ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022369d70}, {0x4e9d7a0, 0xc08e4f3500}, 0xc088ac00c0, 0xc082101c68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022369d70}, 0xc03733e120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07c42e120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07c42e120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806993 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc0606c3410}, {0xc0ae5d7080, 0x2, 0xc00f31ce58}, {0x0, {0x1743010ff62359bd, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc0606c3410}, 0x1866f, 0xc04d7146c0, 0x18, 0xc038af4290) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc0606c3410}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802706 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01e147000, {0x63c85b8, 0xc04e4984b0}, 0xc0890a9700, 0xc0726c7540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e147000, {0x63c85b8, 0xc04e4984b0}, 0xc0890a9700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffce818d9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffce818d9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e498420}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e498420}, {0xc01f2f93f8, 0x660089}, 0xc01f2f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e498420}, {0xc0015221e0}, 0xc0890a9400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e498360}, 0xc0890a9400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}, 0xc039a9fc40, 0xc07db20f30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}, 0x3, 0xc05f65f1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f2f9980}}}, {0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}, 0xc039a9fc40, 0xc05f65f1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e498360}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e498360}, {0x40, 0xc01f2f9b70}, 0xc01f2f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e498360}, {0x4e9d7a0, 0xc0890a9400}, 0xc039a9fc40, 0xc07db20f30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e498360}, 0xc01bf2c1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c4890e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c4890e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219150641 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc03a0118b0, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc03dbc56e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 219810219 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0501bb770}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790592 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc05f452e40}, 0xc01526fcb0, {0x646cc48, 0xc05f452ea0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc05f452e40}, 0xedb7a834e, 0xc033691d40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc05f452e40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06e426b70}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01526fef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06e426b70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 26213 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc011ead7c0, {0xc020318790, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc011ead7c0, {0xc020318790, 0xc015153998, 0xc003502a68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc02ad614a0, {0xc020318790, 0xc003502ae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc02ad614a0}, {0xc020318790, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc013731200, {0xc020318790, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc020318780, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc04783f080, 0xc013731200, {0x0, 0x0}, 0x7fffffff, 0xc003502d90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x63474f8, {0x7fe42ab594c0, 0xc000808f40}, 0x1, {0x0, 0x0}, {0x4cfe5a0, 0xc0342fd818}, 0xc0082db200, 0xc003502d90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc002e3dd40, {0x4cfe5a0, 0xc0342fd818}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc015112dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc015112dc0}, 0xc002496fb0, {0x63c85b8, 0xc02ad61920}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc02ad61920}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800758 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b26c000, {0x63c85b8, 0xc0213edda0}, 0xc045f31e00, 0xc0cdfa7680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc0213edda0}, 0xc045f31e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101657fb9a, 0x0, 0x0}, 0xc01a37af40, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101657fb9a, 0x0, 0x0}, 0xc01a37af40, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0213edd10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0213edd10}, {0xc03af113f8, 0x660089}, 0xc03af11400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0213edd10}, {0xc0015221e0}, 0xc045f31c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0213edbc0}, 0xc045f31c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}, 0xc01f0f9b60, 0xc037048a20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}, 0x3, 0xc07632b880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03af11980}}}, {0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}, 0xc01f0f9b60, 0xc07632b880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0213edbc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0213edbc0}, {0x40, 0xc03af11b70}, 0xc03af11ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0213edbc0}, {0x4e9d7a0, 0xc045f31c00}, 0xc01f0f9b60, 0xc037048a20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0213edbc0}, 0xc03a41b6e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc051228240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc051228240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803859 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018382500, {0x63c85b8, 0xc071a989f0}, 0xc0456ae390, 0x1, 0xc067e710a8, 0x519bbc, 0xc049fe7260, 0xc0132e2e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01e9405a0, {0x63c85b8, 0xc071a989f0}, 0xc01e9405d0, 0xc067e711a8, 0x16, 0x18, 0xffffffffffffffff, 0xc049fe7260, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018382500, {0x63c85b8, 0xc071a989f0}, 0xc049fe7200, {{{{0x0, 0x0}, {0xc046ed6000, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc071a989f0}, 0x6352230, 0x34330a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0181df3b0, {0x63c85b8, 0xc071a989f0}, {0xc049fe70e0, {0x1743011030193d3d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0181df3b0, {0x63c85b8, 0xc071a989f0}, 0xc05b1125a0, 0xc067e719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc071a989f0}, 0x660089, {0xc049fe70e0, {0x1743011030193d3d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0183a4c00, {0x63c85b8, 0xc071a989f0}, 0xc0acd1d500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011030193d3d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011030193d3d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc071a98900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071a98900}, {0xc067e733f8, 0x660089}, 0xc067e73400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc071a98900}, {0x0}, 0xc0acd1d300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc071a98870}, 0xc0acd1d300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}, 0xc061a12a80, 0xc047a27878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}, 0x3, 0xc015b68800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc067e73980}}}, {0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}, 0xc061a12a80, 0xc015b68800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc071a98870}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071a98870}, {0x40, 0xc067e73b70}, 0xc067e73ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc071a98870}, {0x4e9d7a0, 0xc0acd1d300}, 0xc061a12a80, 0xc047a27878) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc071a98870}, 0xc071b89da0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc049fe6ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc049fe6ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810984 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04e7cb1a0}, 0xc0ba4afa28, 0x1, 0xc01a5b70a8, 0x519bbc, 0xc05bf72ba0, 0xc044db4420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04e7cb1a0}, 0x1e0, 0xc01a5b7230, 0x5bf72b40, 0x0, 0x660069, 0xc05bf72ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04e7cb1a0}, 0xc05bf72b40, {{{{0x0, 0x0}, {0xc00bebb440, 0x100}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04e7cb1a0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04e7cb1a0}, {0xc0748bb0e0, {0x174301107142d8e2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04e7cb1a0}, 0xc0b208afa0, 0xc01a5b79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e7cb1a0}, 0x660089, {0xc0748bb0e0, {0x174301107142d8e2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04e7cb1a0}, 0xc007c30a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107142d8e2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107142d8e2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e7cb110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e7cb110}, {0xc01a5b93f8, 0x660089}, 0xc01a5b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e7cb110}, {0x0}, 0xc007c30800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e7cb080}, 0xc007c30800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}, 0xc00aef5520, 0xc031af6b28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}, 0x3, 0xc035b901c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a5b9980}}}, {0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}, 0xc00aef5520, 0xc035b901c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e7cb080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e7cb080}, {0x40, 0xc01a5b9b70}, 0xc01a5b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e7cb080}, {0x4e9d7a0, 0xc007c30800}, 0xc00aef5520, 0xc031af6b28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e7cb080}, 0xc01fa8c300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0748bafc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0748bafc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28614 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0003df278, 0xfc61a) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xadd280) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc0003df260, {0x63c8510, 0xc00b49c200}, 0x327c1) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xde github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x63c85b8, 0xc02cc73470}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x357 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802669 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bfdc400, {0x63c85b8, 0xc04e1bec00}, 0xc020517000, 0xc082d78000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bfdc400, {0x63c85b8, 0xc04e1bec00}, 0xc020517000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102563fbcf, 0x0, 0x0}, 0xc0b0ae0200, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102563fbcf, 0x0, 0x0}, 0xc0b0ae0200, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e1beb70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e1beb70}, {0xc064b953f8, 0x660089}, 0xc064b95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e1beb70}, {0x0}, 0xc020516e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e1beae0}, 0xc020516e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}, 0xc031104420, 0xc07db21b48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}, 0x3, 0xc0cf8ca580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc064b95980}}}, {0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}, 0xc031104420, 0xc0cf8ca580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e1beae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e1beae0}, {0x40, 0xc064b95b70}, 0xc064b95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e1beae0}, {0x4e9d7a0, 0xc020516e00}, 0xc031104420, 0xc07db21b48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e1beae0}, 0xc01bf2dc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0bcf739e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0bcf739e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28438 [select, 716 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0193d9b40) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 219801040 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc061dce000, {0x63c85b8, 0xc099ecb6e0}, 0xc0632b4000, 0xc08e1817c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc099ecb6e0}, 0xc0632b4000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fddb2183d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fddb2183d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc099ecb650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc099ecb650}, {0xc0491d13f8, 0x660089}, 0xc0491d1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc099ecb650}, {0x696d6f6169582220}, 0xc050e1fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc099ecb5c0}, 0xc050e1fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}, 0xc03b73bce0, 0xc027c546a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}, 0x3, 0xc070296600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0491d1980}}}, {0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}, 0xc03b73bce0, 0xc070296600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc099ecb5c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc099ecb5c0}, {0x40, 0xc0491d1b70}, 0xc0491d1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc099ecb5c0}, {0x4e9d7a0, 0xc050e1fd00}, 0xc03b73bce0, 0xc027c546a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc099ecb5c0}, 0xc0b05b8840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04a8c10e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04a8c10e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811203 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc044fe6540}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804683 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc07c79f890}, 0xc0501599f8, 0x1, 0x708017d5b99c66, 0x37e11d600, 0xc06b0184e0, 0xc04cf41380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07c79f890}, 0x120, 0xc091e5b230, 0x6b018480, 0xc001fd9860, 0x18, 0xc06b0184e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc07c79f890}, 0xc06b018480, {{{{0x0, 0x0}, {0xc09298fd40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07c79f890}, 0x6352410, 0x3d0ac9c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc07c79f890}, {0xc06b018360, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc07c79f890}, 0xc041cc12c0, 0xc091e5b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07c79f890}, 0x660089, {0xc06b018360, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc07c79f890}, 0xc00436f300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c79f800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c79f800}, {0xc091e5d3f8, 0x660089}, 0xc091e5d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c79f800}, {0x0}, 0xc00436f100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c79f770}, 0xc00436f100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}, 0xc0320d8c40, 0xc0501599c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}, 0x3, 0xc05ec89600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc091e5d980}}}, {0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}, 0xc0320d8c40, 0xc05ec89600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c79f770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c79f770}, {0x40, 0xc091e5db70}, 0xc091e5dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}, 0x203017, 0x203017) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c79f770}, {0x4e9d7a0, 0xc00436f100}, 0xc0320d8c40, 0xc0501599c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c79f770}, 0xc07062d980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc066b65560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc066b65560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802616 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc041c9ba70}, 0xc046717e78, 0x1, 0xc0553290a8, 0x519bbc, 0xc041b79380, 0xc012cec4e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041c9ba70}, 0x120, 0xc055329230, 0x41b79320, 0x0, 0x660069, 0xc041b79380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc041c9ba70}, 0xc041b79320, {{{{0x0, 0x0}, {0xc09ed7fb00, 0xc4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041c9ba70}, 0x6352410, 0x25ead7c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc041c9ba70}, {0xc041b79200, {0x1743011020dd0e6c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc041c9ba70}, 0xc06550a3c0, 0xc0553299f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041c9ba70}, 0x660089, {0xc041b79200, {0x1743011020dd0e6c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc041c9ba70}, 0xc062531f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020dd0e6c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020dd0e6c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041c9b9b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041c9b9b0}, {0xc05532b3f8, 0x660089}, 0xc05532b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041c9b9b0}, {0xc0015221e0}, 0xc062531d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041c9b920}, 0xc062531d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}, 0xc039a9efa0, 0xc06ba7b5d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}, 0x3, 0xc01d7a4100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05532b980}}}, {0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}, 0xc039a9efa0, 0xc01d7a4100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041c9b920}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041c9b920}, {0x40, 0xc05532bb70}, 0xc05532bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041c9b920}, {0x4e9d7a0, 0xc062531d00}, 0xc039a9efa0, 0xc06ba7b5d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041c9b920}, 0xc050a7f080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc053256b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc053256b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803756 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc0592edf20}, 0xc0547536b0, 0x1, 0x0, 0x1, 0xc047359500, 0xc0449dcf60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0592edf20}, 0x1e0, 0xc0756e8728, 0x473594a0, 0x0, 0x660069, 0xc047359500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc0592edf20}, 0xc0473594a0, {{{{0x0, 0x0}, {0xc01e940000, 0x1b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0592edf20}, 0x6352410, 0x14c75460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc0592edf20}, {0xc0460e07e0, {0x174301102f46acf5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc0592edf20}, 0xc05ad732c0, 0xc04c2f2ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0592edf20}, 0x660089, {0xc0460e07e0, {0x174301102f46acf5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc0592edf20}, 0xc05f1f6000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102f46acf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102f46acf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592ede90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592ede90}, {0xc01fad68f0, 0x660089}, 0xc01fad68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592ede90}, {0xc00102e4e0}, 0xc040e41f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592ede30}, 0xc040e41f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03ff5c060}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811240 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc039b51020}, 0xc07014d500, 0x1, 0x70801810a72e11, 0x37e11d600, 0xc06eb94720, 0xc02c701020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc039b51020}, 0x120, 0xc01639f230, 0x6eb946c0, 0x0, 0x660069, 0xc06eb94720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc039b51020}, 0xc06eb946c0, {{{{0x0, 0x0}, {0xc09c0bd440, 0x108}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc039b51020}, 0x6352410, 0x3594b7d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc039b51020}, {0xc06eb945a0, {0x1743011073cf75a9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc039b51020}, 0xc0a4e9e280, 0xc01639f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc039b51020}, 0x660089, {0xc06eb945a0, {0x1743011073cf75a9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc039b51020}, 0xc0535c1d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011073cf75a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011073cf75a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039b50f90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039b50f90}, {0xc0163a13f8, 0x660089}, 0xc0163a1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039b50f90}, {0x0}, 0xc0535c1a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039b50f00}, 0xc0535c1a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}, 0xc043681ce0, 0xc025b96300) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}, 0x3, 0xc0586d1580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0163a1980}}}, {0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}, 0xc043681ce0, 0xc0586d1580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc039b50f00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039b50f00}, {0x40, 0xc0163a1b70}, 0xc0163a1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc039b50f00}, {0x4e9d7a0, 0xc0535c1a00}, 0xc043681ce0, 0xc025b96300) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc039b50f00}, 0xc01f81e3c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06eb94480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06eb94480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812757 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812496 [runnable]: sync.runtime_SemacquireMutex(0x1, 0x40, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0dca3d1f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04bfea000, {0xc0c85d4540, 0x6e, 0xc0}, {0xc01935ec40, 0x6e, 0x6e}, 0x1, 0xd1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0dca3d088, {0xc0c85d4540, 0x1, 0x6e}, {0xc01935ec40, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0dca3d088, {0xc0c85d4540, 0x6e, 0xc0}, {0xc01935ec40, 0x6e, 0x6e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0dca3c7c0, {0xc01935ec40, 0x6e, 0xc0}, 0x6e, 0x1e) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0dca3c7c0, {0xc0c85d4540, 0x6e, 0xc01935ec40}, {0xc01935ec40, 0x6d, 0xe0}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0dca3c500, {0xc01935ec40, 0x6e, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc087b55228, {{0xc0843c3340, 0x6d, 0xe0}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc08e372c00, {{0xc0843c3340, 0x6d, 0xe0}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x64a6560, 0xc08e372c00}, {{0xc0843c3340, 0x6d, 0xe0}, {0x0, 0x0, 0x0}}, 0x0, 0xc07d409e60) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:856 +0xbf github.com/cockroachdb/cockroach/pkg/storage.mvccPutInternal({0x63c85b8, 0xc04ae6a060}, {0x7fe42e217bc0, 0xc034107000}, {0x64a6560, 0xc08e372c00}, 0xc02addff00, {0xc0843c3340, 0x6d, 0xe0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1287 +0x13d github.com/cockroachdb/cockroach/pkg/storage.mvccPutUsingIter({0x63c85b8, 0xc04ae6a060}, {0x7fe42e217bc0, 0xc034107000}, {0x64a6560, 0xc08e372c00}, 0x0, {0xc0843c3340, 0x6d, 0xe0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1073 +0x1cc github.com/cockroachdb/cockroach/pkg/storage.mvccInitPutUsingIter({0x63c85b8, 0xc04ae6a060}, {0x7fe42e217a50, 0xc034107000}, {0x64a6560, 0xc08e372c00}, 0x4e, {0xc0843c3340, 0x6d, 0xe0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1849 +0x1a5 github.com/cockroachdb/cockroach/pkg/storage.MVCCInitPut({0x63c85b8, 0xc04ae6a060}, {0x7fe42e217a50, 0xc034107000}, 0xc052f3d490, {0xc0843c3340, 0x6d, 0xe0}, {0x1743011082f2a274, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:1814 +0x26a github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.InitPut({_, _}, {_, _}, {{0x64d2498, 0xc023a11800}, {{0x1743011082f2a274, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_init_put.go:39 +0x254 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011082f2a274, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:493 +0x287 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05707e200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:644 +0x190 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:612 +0x2da github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:428 +0x5f3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc023a11800, {0x63c85b8, 0xc04ae6a060}, {0xc06066c168, 0x8}, 0xc05707e200, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:673 +0x1c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x0, {0x63c85b8, _}, {_, _}, _, {{{0x1742da7eeafe3893, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:773 +0xaa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc023a11800, {0x63c85b8, _}, _, _, {{{0x1742da7eeafe3893, 0x0, 0x0}, 0x0, {0x1f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:113 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc023a11800, {0x63c85b8, 0xc04ae6a060}, 0xc05707e200, 0xc08e181b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:167 +0x7b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023a11800, {0x63c85b8, 0xc04ae6a060}, 0xc05707e200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c617fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c617fb0}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c617fb0}, {0xc00102e4e0}, 0xc05707e100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c617f20}, 0xc05707e100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc018086a80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28500 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc007ea07e0, {0x63c8510, 0xc017d656c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:310 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc007ea07e0, {0x63c85b8, 0xc01cc2c090}, 0x20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:335 +0xbb github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x63c85b8, 0xc01cc2c090}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809958 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc041112d20}, 0xc02132aab0, 0x1, 0xc0280070a8, 0x519bbc, 0xc0736656e0, 0xc02c23f920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041112d20}, 0x120, 0xc028007230, 0x73665680, 0x0, 0x660069, 0xc0736656e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc041112d20}, 0xc073665680, {{{{0x0, 0x0}, {0xc05112d440, 0xf5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041112d20}, 0x6352410, 0x51b881d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc041112d20}, {0xc073665560, {0x1743011069147e2c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc041112d20}, 0xc042340be0, 0xc0280079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041112d20}, 0x660089, {0xc073665560, {0x1743011069147e2c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc041112d20}, 0xc05adcbf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011069147e2c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011069147e2c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041112c90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041112c90}, {0xc0280093f8, 0x660089}, 0xc028009400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041112c90}, {0x6f46e1c7deffa11e}, 0xc05adcbd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041112c00}, 0xc05adcbd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}, 0xc03248bfc0, 0xc08a887038) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}, 0x3, 0xc092ef0ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028009980}}}, {0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}, 0xc03248bfc0, 0xc092ef0ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041112c00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041112c00}, {0x40, 0xc028009b70}, 0xc028009ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041112c00}, {0x4e9d7a0, 0xc05adcbd00}, 0xc03248bfc0, 0xc08a887038) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041112c00}, 0xc030bda4e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc073665320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc073665320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811309 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b6f6b40, {0x63c85b8, 0xc07a25b560}, 0xc00b73ea20, 0x1, 0x708018122bde40, 0x37e11d600, 0xc08c3b3c80, 0xc04c0fa240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07a25b560}, 0x1e0, 0xc01a3c5230, 0x8c3b3c20, 0x0, 0x660069, 0xc08c3b3c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b6f6b40, {0x63c85b8, 0xc07a25b560}, 0xc08c3b3c20, {{{{0x0, 0x0}, {0xc063fac360, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07a25b560}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b6f45a0, {0x63c85b8, 0xc07a25b560}, {0xc00b97a000, {0x174301104fbdad95, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b6f45a0, {0x63c85b8, 0xc07a25b560}, 0xc05f1b26e0, 0xc01a3c59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a25b560}, 0x660089, {0xc00b97a000, {0x174301104fbdad95, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b703c00, {0x63c85b8, 0xc07a25b560}, 0xc06d136700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104fbdad95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104fbdad95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a25b4d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a25b4d0}, {0xc01a3c73f8, 0x660089}, 0xc01a3c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a25b4d0}, {0x7fe13d188f60}, 0xc06d136500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a25b440}, 0xc06d136500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}, 0xc00cd97b40, 0xc00b73e990) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}, 0x3, 0xc0b1bf0900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a3c7980}}}, {0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}, 0xc00cd97b40, 0xc0b1bf0900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a25b440}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a25b440}, {0x40, 0xc01a3c7b70}, 0xc01a3c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a25b440}, {0x4e9d7a0, 0xc06d136500}, 0xc00cd97b40, 0xc00b73e990) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a25b440}, 0xc032818f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc00beb9320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc00beb9320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812637 [runnable]: internal/poll.runtime_pollWait(0x7fdb124bcf40, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06d853b80, 0xc0a57e6000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06d853b80, {0xc0a57e6000, 0x205, 0x205}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06d853b80, {0xc0a57e6000, 0x4d91a5, 0xb2eac7d0bfa279c8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc033a92d88, {0xc0a57e6000, 0x4d858f, 0xc04e5e9f90}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04d73afc0, {0xc0a57e6000, 0x65, 0x205}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc060f8e708, {0xc0a57e6000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0787b2278, {0x631b900, 0xc060f8e708}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0787b2000, {0x7fe42ab59310, 0xc04d73afc0}, 0x12835b015807aa98) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0787b2000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).readHandshake(0xc0787b2000) GOROOT/src/crypto/tls/conn.go:1010 +0x6d crypto/tls.(*serverHandshakeStateTLS13).readClientCertificate(0xc078185148) GOROOT/src/crypto/tls/handshake_server_tls13.go:783 +0x65 crypto/tls.(*serverHandshakeStateTLS13).handshake(0xc078185148) GOROOT/src/crypto/tls/handshake_server_tls13.go:74 +0xb6 crypto/tls.(*Conn).serverHandshake(0xc0787b2000, {0x63c8510, 0xc050a07180}) GOROOT/src/crypto/tls/handshake_server.go:54 +0xd0 crypto/tls.(*Conn).handshakeContext(0xc0787b2000, {0x63c8548, 0xc000140068}) GOROOT/src/crypto/tls/conn.go:1453 +0x36f crypto/tls.(*Conn).HandshakeContext(...) GOROOT/src/crypto/tls/conn.go:1396 crypto/tls.(*Conn).Handshake(...) GOROOT/src/crypto/tls/conn.go:1380 crypto/tls.(*Conn).Read(0xc0787b2000, {0xc073d146d8, 0x4, 0xc019549520}) GOROOT/src/crypto/tls/conn.go:1265 +0x65 io.ReadAtLeast({0x7fe456a34450, 0xc0787b2000}, {0xc073d146d8, 0x4, 0x4}, 0x4) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadUntypedMsg(0xc073d146c0, {0x7fe456a34450, 0xc0787b2000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:151 +0x65 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).readVersion(0xc00c689e00, {0x7fe456a34450, 0xc0787b2000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1357 +0x11f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).maybeUpgradeToSecureConn(0xc00c689e00, {0x0, 0xffc}, {0x643d810, 0xc04d73afc0}, 0x2, 0x4d2162f, 0xc078185be0) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1301 +0x367 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c8510, 0xc050a07140}, {0x643d810, 0xc04d73afc0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:807 +0x5a9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc073d14600}, {0x643d810, 0xc04d73afc0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc073d14600}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 17683 [select, 716 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00e0b6120) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00ce1ac60, {0x4cfe6e0, 0xc0044a5500}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc01429bc20, 0xc0110c45b0, 0xc0110c4580) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc01429bc20, {0x4cfe6e0, 0xc0044a5500}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc023dafd20, {0x4cfe6e0, 0xc0044a5500}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc01777c090) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc01777c090}, 0xc005939080, 0xc000cb3540, {0x63c85b8, 0xc023cc2b70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc023cc2b70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812507 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0601e0cd0, {0x63c85b8, 0xc01c616750}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc01c616750}, {0x4f67ad2, 0x10beec0}, 0xc034637fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 28242 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc0087e3b18) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01be137d0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01be137d0, {0x63c8510, 0xc017d653c0}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x63c85b8, 0xc02cafa240}, {0xc0087e3c98, 0x2, 0x4000000000000000}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:193 +0x91 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0013c58c0, {0x63c85b8, 0xc02cafa240}, 0xc02cafa060) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:291 +0x5de github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x63c85b8, 0xc02cafa1b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:215 +0x339 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806938 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc0936c4090}, 0xc067797c08, 0x1, 0x708017f0e3af9e, 0x37e11d600, 0xc0590b8de0, 0xc0a07b3920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0936c4090}, 0x120, 0xc004ce3230, 0x590b8d80, 0x0, 0x660069, 0xc0590b8de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc0936c4090}, 0xc0590b8d80, {{{{0x0, 0x0}, {0xc031d92b40, 0x27}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0936c4090}, 0x6352410, 0x5449b630) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc0936c4090}, {0xc0590b8c60, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc0936c4090}, 0xc00aae88c0, 0xc004ce39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0936c4090}, 0x660089, {0xc0590b8c60, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0936c4090}, 0xc078eff300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0936c4000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0936c4000}, {0xc004ce53f8, 0x660089}, 0xc004ce5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0936c4000}, {0x4086e661e4581d6f}, 0xc078eff100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fcd3f50}, 0xc078eff100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}, 0xc08fb7dd20, 0xc067797bd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}, 0x3, 0xc08efba8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004ce5980}}}, {0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}, 0xc08fb7dd20, 0xc08efba8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06fcd3f50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fcd3f50}, {0x40, 0xc004ce5b70}, 0xc004ce5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06fcd3f50}, {0x4e9d7a0, 0xc078eff100}, 0xc08fb7dd20, 0xc067797bd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06fcd3f50}, 0xc0402751a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590b8900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590b8900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219459888 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0397eb810, {0x63c85b8, 0xc07cdb8210}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc03fb00000, {0x63c85b8, 0xc07cdb8210}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc03fb00000, {0x63c8510, 0xc051ef5580}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc03fb00000, {0x63c8510, 0xc051ef5580}, {0x63f7db8, 0x9bf6f48}, 0xc07d16b088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc03fb00000, {0x63c85b8, 0xc0ce31a780}, {{0x63f7db8, 0x9bf6f48}, {0xc0582fa2e8, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc03fb00000, {0x63c85b8, 0xc0ce31a780}, {{0x63f7db8, 0x9bf6f48}, {0xc0582fa2e8, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0582fa2e8, 0x6}, 0x0, 0x0}, {0xbb1619b, 0xedb7a834f, 0x0}, {0xbb169dc, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03fb00000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03fb00000, {0x63c8510, 0xc0d29de000}, 0xc0596dddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0898b1990) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0d29de000}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0898b1990) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219790481 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc04781a8f0, {0x63c85b8, 0xc03bc92ed0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc019e3e000, {0x63c85b8, 0xc03bc92ed0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc019e3e000, {0x63c8510, 0xc066f92580}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc019e3e000, {0x63c8510, 0xc066f92580}, {0x63f7db8, 0x9bf6f48}, 0xc078165088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc019e3e000, {0x63c85b8, 0xc067bb2360}, {{0x63f7db8, 0x9bf6f48}, {0xc07a51d17d, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc019e3e000, {0x63c85b8, 0xc067bb2360}, {{0x63f7db8, 0x9bf6f48}, {0xc07a51d17d, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc07a51d17d, 0x6}, 0x0, 0x0}, {0x2ee4ef80, 0xedb7a834f, 0x0}, {0x2ee4f67e, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc019e3e000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc019e3e000, {0x63c8510, 0xc0234d4340}, 0xc0704dddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc087fe6d50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0234d4340}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc087fe6d50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219748402 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc095eb2dc0, {0x63c85b8, 0xc02822be00}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc074a69300, {0x63c85b8, 0xc02822be00}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc074a69300, {0x63c8510, 0xc02eaf5ac0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc074a69300, {0x63c8510, 0xc02eaf5ac0}, {0x63f7db8, 0x9bf6f48}, 0xc0846bf088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc074a69300, {0x63c85b8, 0xc054c1e5d0}, {{0x63f7db8, 0x9bf6f48}, {0xc037ed9dd4, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc074a69300, {0x63c85b8, 0xc054c1e5d0}, {{0x63f7db8, 0x9bf6f48}, {0xc037ed9dd4, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc037ed9dd4, 0x6}, 0x0, 0x0}, {0x171105c1, 0xedb7a8350, 0x0}, {0x17110d62, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc074a69300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc074a69300, {0x63c8510, 0xc041bfacc0}, 0xc08da99dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc056311990) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc041bfacc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc056311990) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802717 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020b11c00, {0x63c85b8, 0xc06f5e0c00}, 0xc0a67c9600, 0xc0627cc780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b11c00, {0x63c85b8, 0xc06f5e0c00}, 0xc0a67c9600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102522fef3, 0x0, 0x0}, 0xc064844fe0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102522fef3, 0x0, 0x0}, 0xc064844fe0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f5e0b70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f5e0b70}, {0xc076115918, 0x660089}, 0xc076115920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f5e0b70}, {0xc00102e4e0}, 0xc0a67c9500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f5e0b10}, 0xc0a67c9500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc026e9bd40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06f5e0a80}, {0xc00dfd36b0, 0x24}, 0x46c7b3d81, 0xc06366bd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc0786a2840}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc06f5e0a80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811909 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06d400, {0x63c85b8, 0xc015262ea0}, 0xc0aa691a00, 0xc02a864320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06d400, {0x63c85b8, 0xc015262ea0}, 0xc0aa691a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b1cf270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b1cf270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc015262e10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015262e10}, {0xc01f2f93f8, 0x660089}, 0xc01f2f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc015262e10}, {0x7fdd4a3e6eb8}, 0xc0aa691800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc015262d50}, 0xc0aa691800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}, 0xc0bb7eaae0, 0xc0d3545728) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}, 0x3, 0xc076b1a080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f2f9980}}}, {0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}, 0xc0bb7eaae0, 0xc076b1a080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc015262d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015262d50}, {0x40, 0xc01f2f9b70}, 0xc01f2f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc015262d50}, {0x4e9d7a0, 0xc0aa691800}, 0xc0bb7eaae0, 0xc0d3545728) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc015262d50}, 0xc0b9a16960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04094cc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04094cc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812658 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808088 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc041f14d80}, 0xc050547080, 0x1, 0xc00c0b90a8, 0x519bbc, 0xc08aa8bc80, 0xc065130f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc041f14d80}, 0x1e0, 0xc00c0b9230, 0x8aa8bc20, 0x0, 0x660069, 0xc08aa8bc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc041f14d80}, 0xc08aa8bc20, {{{{0x0, 0x0}, {0xc065fcf8c0, 0x3b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc01e533e60, {0x63c85b8, 0xc041f14d80}, 0xc081b23ed0, 0xc0b9428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc041f14d80}, {0xc052408a20, {0x174301105b2923dd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc041f14d80}, 0xc066746a00, 0xc00c0b99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041f14d80}, 0x660089, {0xc052408a20, {0x174301105b2923dd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc041f14d80}, 0xc083e35a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b2923dd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b2923dd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041f14cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041f14cf0}, {0xc00c0bb3f8, 0x660089}, 0xc00c0bb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041f14cf0}, {0x7fe348971310}, 0xc083e35800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041f14c60}, 0xc083e35800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}, 0xc0734a5a00, 0xc0274ad6b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}, 0x3, 0xc015b31340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00c0bb980}}}, {0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}, 0xc0734a5a00, 0xc015b31340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041f14c60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041f14c60}, {0x40, 0xc00c0bbb70}, 0xc00c0bbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041f14c60}, {0x4e9d7a0, 0xc083e35800}, 0xc0734a5a00, 0xc0274ad6b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041f14c60}, 0xc08e058fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc052408900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc052408900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806552 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021c5b400, {0x63c85b8, 0xc08c8d6ba0}, 0xc027813368, 0x1, 0x0, 0x1, 0xc053c4f020, 0xc04d420de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08c8d6ba0}, 0x120, 0xc005c1a728, 0x53c4efc0, 0x0, 0x660069, 0xc053c4f020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021c5b400, {0x63c85b8, 0xc08c8d6ba0}, 0xc053c4efc0, {{{{0x0, 0x0}, {0xc056428000, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08c8d6ba0}, 0x6352410, 0x63fdcb38) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c42d20, {0x63c85b8, 0xc08c8d6ba0}, {0xc00cd8b320, {0x174301104e6522ec, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c42d20, {0x63c85b8, 0xc08c8d6ba0}, 0xc05b634000, 0xc089d54ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08c8d6ba0}, 0x660089, {0xc00cd8b320, {0x174301104e6522ec, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c87400, {0x63c85b8, 0xc08c8d6ba0}, 0xc085d7f900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e6522ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e6522ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08c8d6b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08c8d6b10}, {0xc0c14048f0, 0x660089}, 0xc0c14048f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08c8d6b10}, {0xc00102e4e0}, 0xc085d7f800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08c8d6ab0}, 0xc085d7f800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc08bb70810}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219176499 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff35c10, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc082898e00, 0xc052efc000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc082898e00, {0xc052efc000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc082898e00, {0xc052efc000, 0x735, 0xc0351e7400}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc041cbbcc8, {0xc052efc000, 0x7eda79, 0xc052efc005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02d28d320, {0xc052efc000, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04033e990, {0xc052efc000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc052aee278, {0x631b900, 0xc04033e990}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc052aee000, {0x7fe42ab59310, 0xc02d28d320}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc052aee000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc052aee000, {0xc0aaf4c000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0395ce5d0, {0xc0aaf4c000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc044044b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc044044b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0440450e0, {0x634aac0, 0xc044044b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc044044a80, 0xc084daad50, 0xc0678cf42b, 0xc0678cf42d, {0x635bec0, 0xc044f96240}, 0xc000d6ae00, 0xc064745fe4, 0xc0678cf429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc044044a80, {0x63c85b8, 0xc06187d6b0}, 0xc084daad40, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc00868e03b, 0x8}}, 0x0, 0xc06187d650, 0xc06187d680, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06187d6b0}, {0x643d810, 0xc02d28d320}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc043755f80}, {0x643d810, 0xc02d28d320}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc043755f80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803232 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc016d66000}, 0xc059010500, 0xc070c72460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc016d66000}, 0xc059010500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc46955, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc46955, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06dea7f50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dea7f50}, {0xc065d1d3f8, 0x660089}, 0xc065d1d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06dea7f50}, {0xc0015221e0}, 0xc059b2bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06dea7ec0}, 0xc059b2bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}, 0xc02062b380, 0xc07b4d1230) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}, 0x3, 0xc040f35940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc065d1d980}}}, {0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}, 0xc02062b380, 0xc040f35940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06dea7ec0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dea7ec0}, {0x40, 0xc065d1db70}, 0xc065d1dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06dea7ec0}, {0x4e9d7a0, 0xc059b2bf00}, 0xc02062b380, 0xc07b4d1230) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06dea7ec0}, 0xc02b0aae40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc038ab0d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc038ab0d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811863 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01ded1400, {0x63c85b8, 0xc0763006f0}, 0xc076dff100, 0xc06b47af00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ded1400, {0x63c85b8, 0xc0763006f0}, 0xc076dff100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ea6c77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ea6c77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076300630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076300630}, {0xc015e813f8, 0x660089}, 0xc015e81400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076300630}, {0x7fe3e21dbb88}, 0xc076dfed00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0763005a0}, 0xc076dfed00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}, 0xc044a379e0, 0xc0457057e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}, 0x3, 0xc0875f4d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc015e81980}}}, {0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}, 0xc044a379e0, 0xc0875f4d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0763005a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0763005a0}, {0x40, 0xc015e81b70}, 0xc015e81ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0763005a0}, {0x4e9d7a0, 0xc076dfed00}, 0xc044a379e0, 0xc0457057e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0763005a0}, 0xc0209e1740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc059511d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc059511d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811291 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0772dcae0}, 0xc0cdaf2c18, 0x1, 0xc0179650a8, 0x519bbc, 0xc012c3d920, 0xc0308d76e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0772dcae0}, 0x120, 0xc017965230, 0x12c3d8c0, 0x0, 0x660069, 0xc012c3d920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0772dcae0}, 0xc012c3d8c0, {{{{0x0, 0x0}, {0xc0267bf8c0, 0x117}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0772dcae0}, 0x6352410, 0x55bf3108) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0772dcae0}, {0xc012c3d7a0, {0x1743011075b17655, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0772dcae0}, 0xc00b5b2aa0, 0xc0179659f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0772dcae0}, 0x660089, {0xc012c3d7a0, {0x1743011075b17655, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0772dcae0}, 0xc0be439500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011075b17655, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011075b17655, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0772dca50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0772dca50}, {0xc0179673f8, 0x660089}, 0xc017967400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0772dca50}, {0x7fe323ba83b0}, 0xc0be439300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0772dc9c0}, 0xc0be439300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}, 0xc05ba78980, 0xc0cdaf3698) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}, 0x3, 0xc05279b240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017967980}}}, {0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}, 0xc05ba78980, 0xc05279b240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0772dc9c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0772dc9c0}, {0x40, 0xc017967b70}, 0xc017967ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0772dc9c0}, {0x4e9d7a0, 0xc0be439300}, 0xc05ba78980, 0xc0cdaf3698) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0772dc9c0}, 0xc04f640cc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc012c3d680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc012c3d680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805136 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc02a729d40}, 0xc0724a6540, 0x1, 0x0, 0x1, 0xc072c9c2a0, 0xc04cfe3260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02a729d40}, 0x120, 0xc079456728, 0x72c9c240, 0xc001fec1a0, 0x18, 0xc072c9c2a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc02a729d40}, 0xc072c9c240, {{{{0x0, 0x0}, {0xc05748cb40, 0x22}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02a729d40}, 0x6352410, 0x71c3ea20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc02a729d40}, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc02a729d40}, 0xc054f81ea0, 0xc076674ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02a729d40}, 0x660089, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc02a729d40}, 0xc063d40d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02a729aa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02a729aa0}, {0xc09359a8f0, 0x660089}, 0xc09359a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02a729aa0}, {0xc00102e4e0}, 0xc063d40c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02a729a40}, 0xc063d40c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02a7293b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801606 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc045247c80}, 0xc06ab8eb00, 0xc04caf5b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc045247c80}, 0xc06ab8eb00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101aa4b463, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101aa4b463, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045247bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045247bf0}, {0xc051cfb3f8, 0x660089}, 0xc051cfb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045247bf0}, {0x7fe426bcf5e8}, 0xc06ab8e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045247b30}, 0xc06ab8e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}, 0xc072f078e0, 0xc0267594e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}, 0x3, 0xc04a4a6e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc051cfb980}}}, {0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}, 0xc072f078e0, 0xc04a4a6e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045247b30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045247b30}, {0x40, 0xc051cfbb70}, 0xc051cfbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045247b30}, {0x4e9d7a0, 0xc06ab8e800}, 0xc072f078e0, 0xc0267594e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045247b30}, 0xc04f41d1a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc048e71b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc048e71b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808149 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e034f00, {0x63c85b8, 0xc0222909f0}, 0xc08324a678, 0x1, 0xc006a1f0a8, 0x519bbc, 0xc072f77ec0, 0xc07a90dda0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0222909f0}, 0x120, 0xc006a1f230, 0x72f77e60, 0x0, 0x660069, 0xc072f77ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e034f00, {0x63c85b8, 0xc0222909f0}, 0xc072f77e60, {{{{0x0, 0x0}, {0xc03cfe8480, 0xa}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0222909f0}, 0x6352410, 0x2eda49d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd310, {0x63c85b8, 0xc0222909f0}, {0xc072f77d40, {0x174301105aa31fd2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd310, {0x63c85b8, 0xc0222909f0}, 0xc05292ec80, 0xc006a1f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0222909f0}, 0x660089, {0xc072f77d40, {0x174301105aa31fd2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc0222909f0}, 0xc072374300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aa31fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aa31fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022290900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022290900}, {0xc006a213f8, 0x660089}, 0xc006a21400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022290900}, {0x7fe3b58493a8}, 0xc07babdf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022290690}, 0xc07babdf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}, 0xc078d8b880, 0xc014e588e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}, 0x3, 0xc046a21680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006a21980}}}, {0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}, 0xc078d8b880, 0xc046a21680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022290690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022290690}, {0x40, 0xc006a21b70}, 0xc006a21ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022290690}, {0x4e9d7a0, 0xc07babdf00}, 0xc078d8b880, 0xc014e588e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022290690}, 0xc026c28780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07b21a120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07b21a120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805758 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bbe1680, {0x63c85b8, 0xc0769dcd80}, 0xc04cebb8a8, 0x1, 0xc006e070a8, 0x519bbc, 0xc0518c7aa0, 0xc03272c9c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0769dcd80}, 0x1e0, 0xc006e07230, 0x518c7a40, 0x0, 0x660069, 0xc0518c7aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bbe1680, {0x63c85b8, 0xc0769dcd80}, 0xc0518c7a40, {{{{0x0, 0x0}, {0xc02fcbeea0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xf3b92c44b9c312be, {0x63c85b8, 0xc0769dcd80}, 0xcabba0b175af1b14, 0xbe432584) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bae1db0, {0x63c85b8, 0xc0769dcd80}, {0xc038f15c20, {0x17430110026baf99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bae1db0, {0x63c85b8, 0xc0769dcd80}, 0xc0761edae0, 0xc006e079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0769dcd80}, 0x660089, {0xc038f15c20, {0x17430110026baf99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bca4c00, {0x63c85b8, 0xc0769dcd80}, 0xc053477600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110026baf99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110026baf99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0769dccf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0769dccf0}, {0xc006e093f8, 0x660089}, 0xc006e09400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0769dccf0}, {0x90733d111bee1e93}, 0xc053477400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0769dcc30}, 0xc053477400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}, 0xc0915abb20, 0xc08a7ea828) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}, 0x3, 0xc093ea3380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006e09980}}}, {0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}, 0xc0915abb20, 0xc093ea3380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0769dcc30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0769dcc30}, {0x40, 0xc006e09b70}, 0xc006e09ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0769dcc30}, {0x4e9d7a0, 0xc053477400}, 0xc0915abb20, 0xc08a7ea828) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0769dcc30}, 0xc0274e4900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc038f15b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc038f15b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219459980 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff362a0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc097871400, 0xc0455e7000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc097871400, {0xc0455e7000, 0x2b83, 0x2b83}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc097871400, {0xc0455e7000, 0x2b83, 0xc085a50640}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc003f51098, {0xc0455e7000, 0x7eda79, 0xc0455e7005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02c7687e0, {0xc0455e7000, 0xc07196cc60, 0x2b83}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02f7539c8, {0xc0455e7000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0d03a2cf8, {0x631b900, 0xc02f7539c8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0d03a2a80, {0x7fe42ab59310, 0xc02c7687e0}, 0x2b83) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0d03a2a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0d03a2a80, {0xc06ec27000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02554a540, {0xc06ec27000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03f164088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03f164088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03f164660, {0x634aac0, 0xc03f164088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03f164000, 0xc0898b1970, 0xc0854c342b, 0xc0854c342d, {0x635bec0, 0xc016cdf200}, 0xc000d6ae00, 0xc097e1b588, 0xc0854c3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03f164000, {0x63c85b8, 0xc04ea61fb0}, 0xc0898b1960, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09993c037, 0x8}}, 0x0, 0xc04ea61f50, 0xc04ea61f80, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04ea61fb0}, {0x643d810, 0xc02c7687e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0782733b0}, {0x643d810, 0xc02c7687e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0782733b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28389 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002944b60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 219719155 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff35a30, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc068030680, 0xc052a4f800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc068030680, {0xc052a4f800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc068030680, {0xc052a4f800, 0x735, 0xc05a317cc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc071c3f3b0, {0xc052a4f800, 0x7eda79, 0xc052a4f805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0705d2a80, {0xc052a4f800, 0x6, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc01c104ba0, {0xc052a4f800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc054d49af8, {0x631b900, 0xc01c104ba0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc054d49880, {0x7fe42ab59310, 0xc0705d2a80}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc054d49880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc054d49880, {0xc0a54d5000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03c5649a8, {0xc0a54d5000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05646eb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05646eb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05646f0e0, {0x634aac0, 0xc05646eb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05646ea80, 0xc0626c70f0, 0xc08304b42b, 0xc08304b42d, {0x635bec0, 0xc0b1552ab0}, 0xc000d6ae00, 0xc0254c6588, 0xc08304b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05646ea80, {0x63c85b8, 0xc0398e9d10}, 0xc0626c70e0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08873c047, 0x8}}, 0x0, 0xc0398e9cb0, 0xc0398e9ce0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0398e9d10}, {0x643d810, 0xc0705d2a80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0155e3920}, {0x643d810, 0xc0705d2a80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0155e3920}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803014 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016e7a140, {0x63c85b8, 0xc0ce8f7800}, 0xc072e32ed0, 0x1, 0xc0054330a8, 0x519bbc, 0xc041519020, 0xc060f6db60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0ce8f7800}, 0x120, 0xc005433230, 0x41518fc0, 0x0, 0x660069, 0xc041519020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016e7a140, {0x63c85b8, 0xc0ce8f7800}, 0xc041518fc0, {{{{0x0, 0x0}, {0xc058f6a900, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0ce8f7800}, 0x6352410, 0x3080e600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016e743c0, {0x63c85b8, 0xc0ce8f7800}, {0xc041518ea0, {0x174301102573765d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016e743c0, {0x63c85b8, 0xc0ce8f7800}, 0xc067f6c6e0, 0xc0054339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0ce8f7800}, 0x660089, {0xc041518ea0, {0x174301102573765d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc015909c00, {0x63c85b8, 0xc0ce8f7800}, 0xc00c23a000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102573765d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102573765d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0ce8f7770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ce8f7770}, {0xc0054353f8, 0x660089}, 0xc005435400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0ce8f7770}, {0x7fdd4a3cf118}, 0xc0aa4ade00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0ce8f76e0}, 0xc0aa4ade00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}, 0xc03cfbcc00, 0xc035a1eb28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}, 0x3, 0xc0890f8800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005435980}}}, {0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}, 0xc03cfbcc00, 0xc0890f8800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0ce8f76e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ce8f76e0}, {0x40, 0xc005435b70}, 0xc005435ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0ce8f76e0}, {0x4e9d7a0, 0xc0aa4ade00}, 0xc03cfbcc00, 0xc035a1eb28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0ce8f76e0}, 0xc029a70960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041518d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041518d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804979 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bc66c80, {0x63c85b8, 0xc011ac0e70}, 0xc06508cd68, 0x1, 0xc07a1ed0a8, 0x519bbc, 0xc06f893da0, 0xc076b09c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc011ac0e70}, 0x120, 0xc07a1ed230, 0x6f893d40, 0xc001fed040, 0x18, 0xc06f893da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bc66c80, {0x63c85b8, 0xc011ac0e70}, 0xc06f893d40, {{{{0x0, 0x0}, {0xc06826e900, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc011ac0e70}, 0x6352410, 0x71c3e4d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bc685f0, {0x63c85b8, 0xc011ac0e70}, {0xc06f893c20, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bc685f0, {0x63c85b8, 0xc011ac0e70}, 0xc07839d7c0, 0xc07a1ed9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc011ac0e70}, 0x660089, {0xc06f893c20, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bf90c00, {0x63c85b8, 0xc011ac0e70}, 0xc05075e200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc011ac0de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc011ac0de0}, {0xc07a1ef3f8, 0x660089}, 0xc07a1ef400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc011ac0de0}, {0x7fe3b596e6a8}, 0xc05075e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc011ac0d50}, 0xc05075e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}, 0xc06c96a7e0, 0xc0724a6408) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}, 0x3, 0xc066c89d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07a1ef980}}}, {0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}, 0xc06c96a7e0, 0xc066c89d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc011ac0d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc011ac0d50}, {0x40, 0xc07a1efb70}, 0xc07a1efab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}, 0x203019, 0x203019) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc011ac0d50}, {0x4e9d7a0, 0xc05075e000}, 0xc06c96a7e0, 0xc0724a6408) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc011ac0d50}, 0xc0bf444780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85d0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85d0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808519 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc067a8bbf0}, 0xc039d976b0, 0x1, 0xc0072330a8, 0x519bbc, 0xc064a2f800, 0xc05baee720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc067a8bbf0}, 0x120, 0xc007233230, 0x64a2f7a0, 0x0, 0x660069, 0xc064a2f800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc067a8bbf0}, 0xc064a2f7a0, {{{{0x0, 0x0}, {0xc05c483200, 0x125}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc067a8bbf0}, 0x6352410, 0x43390610) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc067a8bbf0}, {0xc064a2f680, {0x174301105f102c3b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc067a8bbf0}, 0xc08e181a40, 0xc0072339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067a8bbf0}, 0x660089, {0xc064a2f680, {0x174301105f102c3b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc067a8bbf0}, 0xc059a07500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105f102c3b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105f102c3b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067a8bb60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067a8bb60}, {0xc0072353f8, 0x660089}, 0xc007235400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067a8bb60}, {0x0}, 0xc059a07300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067a8bad0}, 0xc059a07300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}, 0xc035707020, 0xc020bd5218) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}, 0x3, 0xc0672b6fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007235980}}}, {0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}, 0xc035707020, 0xc0672b6fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc067a8bad0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067a8bad0}, {0x40, 0xc007235b70}, 0xc007235ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc067a8bad0}, {0x4e9d7a0, 0xc059a07300}, 0xc035707020, 0xc020bd5218) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc067a8bad0}, 0xc06d21c420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc064a2f560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc064a2f560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803741 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01edcb400, {0x63c85b8, 0xc0446fea20}, 0xc070957500, 0xc02cd02b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0446fea20}, 0xc070957500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fa26af4, 0x0, 0x0}, 0xc0386b74b0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fa26af4, 0x0, 0x0}, 0xc0386b74b0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0446fe990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0446fe990}, {0xc04ed173f8, 0x660089}, 0xc04ed17400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0446fe990}, {0x7fe13d162a38}, 0xc070957300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0446fe8d0}, 0xc070957300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}, 0xc0334e3860, 0xc026cf88d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}, 0x3, 0xc03cd25380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04ed17980}}}, {0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}, 0xc0334e3860, 0xc03cd25380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0446fe8d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0446fe8d0}, {0x40, 0xc04ed17b70}, 0xc04ed17ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0446fe8d0}, {0x4e9d7a0, 0xc070957300}, 0xc0334e3860, 0xc026cf88d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0446fe8d0}, 0xc02fee2060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0471fe240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0471fe240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 27338 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc000e67b18) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01d2301b0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01d2301b0, {0x63c8510, 0xc02aec1040}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x63c85b8, 0xc02cb31c50}, {0xc000e67c98, 0x2, 0x4000000000000000}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:193 +0x91 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00148f760, {0x63c85b8, 0xc02cb31c50}, 0xc02ccd8ae0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:291 +0x5de github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x63c85b8, 0xc02cb31bc0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:215 +0x339 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219784728 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc079acff10, 0x46) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0910c4a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0910c50e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0ce8c6600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0ce8c6600, {0x63c8510, 0xc079acff40}, 0xc0764bfdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07cc224f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc079acff40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07cc224f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219806533 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc086ba96e0}, 0xc08cc8edf8, 0x1, 0x0, 0x1, 0xc0d08ae600, 0xc086d96180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc086ba96e0}, 0x120, 0xc00c4ec728, 0xd08ae5a0, 0x0, 0x660069, 0xc0d08ae600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc086ba96e0}, 0xc0d08ae5a0, {{{{0x0, 0x0}, {0xc09ecf18c0, 0x106}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc086ba96e0}, 0x6352410, 0x5449a628) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc086ba96e0}, {0xc01dc307e0, {0x174301104eb46d21, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc086ba96e0}, 0xc07882ed20, 0xc04c2f2ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc086ba96e0}, 0x660089, {0xc01dc307e0, {0x174301104eb46d21, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc086ba96e0}, 0xc037d5ae00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104eb46d21, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104eb46d21, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086ba9650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086ba9650}, {0xc0abc508f0, 0x660089}, 0xc0abc508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086ba9650}, {0xc00102e4e0}, 0xc037d5ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086ba95f0}, 0xc037d5ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc088d4d800}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805335 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0713b7050}, 0xc017750c78, 0x1, 0xc063b3b0a8, 0x519bbc, 0xc0595dd500, 0xc059bb7500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0713b7050}, 0x1e0, 0xc063b3b230, 0x595dd4a0, 0xc001ff2000, 0x18, 0xc0595dd500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0713b7050}, 0xc0595dd4a0, {{{{0x0, 0x0}, {0xc05aa42480, 0x55}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0713b7050}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0713b7050}, {0xc05b0d0480, {0x174301103f2af520, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0713b7050}, 0xc0c75d1540, 0xc063b3b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0713b7050}, 0x660089, {0xc05b0d0480, {0x174301103f2af520, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0713b7050}, 0xc0984c4a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f2af520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f2af520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0713b6fc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0713b6fc0}, {0xc063b3d3f8, 0x660089}, 0xc063b3d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0713b6fc0}, {0xc0015221e0}, 0xc0984c4800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0713b6f30}, 0xc0984c4800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}, 0xc064a6c460, 0xc017750e70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}, 0x3, 0xc035d80f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc063b3d980}}}, {0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}, 0xc064a6c460, 0xc035d80f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0713b6f30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0713b6f30}, {0x40, 0xc063b3db70}, 0xc063b3dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}, 0x20300d, 0x20300d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0713b6f30}, {0x4e9d7a0, 0xc0984c4800}, 0xc064a6c460, 0xc017750e70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0713b6f30}, 0xc020f184e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05b0d0360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05b0d0360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812391 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc07a157fb0}, 0xc00a65a948, 0x1, 0xc020cb50a8, 0x519bbc, 0xc0441e2b60, 0xc0182920c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01d122c60, {0x63c85b8, 0xc07a157fb0}, 0xc01d122c98, 0xc020cb51a8, 0x40, 0x60, 0xffffffffffffffff, 0xc0441e2b60, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc07a157fb0}, 0xc0454bc0c0, {{{{0xc073da7680, 0x5}, {0xc060fc7560, 0x13}}, {{0xc0afc31440, 0x1a}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07a157fb0}, 0x63521a0, 0x65f8b128) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc07a157fb0}, {0xc013054240, {0x17430110174a246b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc07a157fb0}, 0xc04f428820, 0xc020cb59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a157fb0}, 0x660089, {0xc013054240, {0x17430110174a246b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc07a157fb0}, 0xc0839f4300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110174a246b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110174a246b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a157f20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a157f20}, {0xc020cb73f8, 0x660089}, 0xc020cb7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a157f20}, {0xc025e67680}, 0xc0839f4000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a157e60}, 0xc0839f4000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}, 0xc0612025c0, 0xc073113458) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}, 0x3, 0xc0666a4840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc020cb7980}}}, {0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}, 0xc0612025c0, 0xc0666a4840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a157e60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a157e60}, {0x40, 0xc020cb7b70}, 0xc020cb7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a157e60}, {0x4e9d7a0, 0xc0839f4000}, 0xc0612025c0, 0xc073113458) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a157e60}, 0xc050f79f20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013054120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013054120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799948 [runnable]: sync.runtime_SemacquireMutex(0xc08c3e66a0, 0x8f, 0x118429a) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x1194e0b, 0xc08c3e6500, 0x11948b8) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x33, 0x97, 0x34, 0x2b) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc021f8f8c0, {0x7ffc973, 0x3071f}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc08fb99680, {0xffffffffffffff15, 0xc08c3e66d0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc08fb99680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc08fb99680, {0xc02e513380, 0x2b, 0x71}, {0xc0289350e0, 0x2b, 0x97}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc090e4b998, {0xc02e513380, 0x2b, 0x71}, {0xc0289350e0, 0x2b, 0x97}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc090e4a2c0, {0xc0289350e0, 0x97, 0x71}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc090e4a2c0, {0xc02e513380, 0x2b, 0xc0289350e0}, {0xc0289350e0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc090e4a000, {0xc0289350e0, 0x2b, 0x97}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc054d79708, {{0xc066d16660, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04cdfca00, {{0xc066d16660, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04d45f400, {0x63c85b8, 0xc068429680}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc068429680}, {0x64a6560, 0xc04cdfca00}, {0xc066d16660, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc068429680}, {0x6466ed8, 0xc054d79700}, {0xc066d16660, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc015393f50}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc060b52200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209ef800, {0x63c85b8, 0xc068429680}, 0xc060b52200, 0xc086642140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209ef800, {0x63c85b8, 0xc068429680}, 0xc060b52200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0684295c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0684295c0}, {0xc0031068f0, 0x660089}, 0xc0031068f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0684295c0}, {0xc00102e4e0}, 0xc060b52100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc068429560}, 0xc060b52100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc024122540}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809496 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01cededc0, {0x63c85b8, 0xc0403abd70}, 0xc0cf896d08, 0x1, 0x0, 0x1, 0xc07cbf5da0, 0xc044d574a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0403abd70}, 0x120, 0xc01aa58728, 0x7cbf5d40, 0x0, 0x660069, 0xc07cbf5da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01cededc0, {0x63c85b8, 0xc0403abd70}, 0xc07cbf5d40, {{{{0x0, 0x0}, {0xc07ab80000, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0403abd70}, 0x6352410, 0x57085300) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01ceee640, {0x63c85b8, 0xc0403abd70}, {0xc07cbf58c0, {0x17430110645c17f1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01ceee640, {0x63c85b8, 0xc0403abd70}, 0xc01cc9a780, 0xc07e20eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0403abd70}, 0x660089, {0xc07cbf58c0, {0x17430110645c17f1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01cfb1000, {0x63c85b8, 0xc0403abd70}, 0xc08ae25800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110645c17f1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110645c17f1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0403abce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0403abce0}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0403abce0}, {0xc00102e4e0}, 0xc08ae25600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0403abc80}, 0xc08ae25600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0403ab5f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800446 [semacquire]: sync.runtime_SemacquireMutex(0xc0958726a0, 0x8f, 0x3b81) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc095872538, 0x2a, 0x2a) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0xc095872558, 0x114acc7, 0xc0abd7c480) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0254bc2c0, {0x7fc09ad, 0x303ba}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04f94c480, {0xffffffffffffff15, 0xc0958726d0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc04f94c480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04f94c480, {0xc03642bd80, 0x2b, 0x79}, {0xc042a27570, 0x2b, 0x68}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092d9f998, {0xc03642bd80, 0x2b, 0x79}, {0xc042a27570, 0x2b, 0x68}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092d9e2c0, {0xc042a27570, 0x68, 0x79}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092d9e2c0, {0xc03642bd80, 0x2b, 0xc042a27570}, {0xc042a27570, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092d9e000, {0xc042a27570, 0x2b, 0x68}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc052cea588, {{0xc047ee66c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04eeb9400, {{0xc047ee66c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc053137000, {0x63c85b8, 0xc032ab4900}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc032ab4900}, {0x64a6560, 0xc04eeb9400}, {0xc047ee66c0, 0x4d545, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc032ab4900}, {0x6466ed8, 0xc052cea580}, {0xc047ee66c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9b050}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0975c3c00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020af2800, {0x63c85b8, 0xc032ab4900}, 0xc0975c3c00, 0xc039b4aaa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020af2800, {0x63c85b8, 0xc032ab4900}, 0xc0975c3c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc032ab4840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc032ab4840}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc032ab4840}, {0xc00102e4e0}, 0xc0975c3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc032ab47e0}, 0xc0975c3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06b4b8ab0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806710 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc011da9800, {0x63c85b8, 0xc08db06600}, 0xc063997b00, 0xc03ab4c0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc011da9800, {0x63c85b8, 0xc08db06600}, 0xc063997b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105099f20f, 0x0, 0x0}, 0xc08b3d49c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105099f20f, 0x0, 0x0}, 0xc08b3d49c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08db06540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08db06540}, {0xc09363c8f0, 0x660089}, 0xc09363c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08db06540}, {0xc00102e4e0}, 0xc063997a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08db064e0}, 0xc063997a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc08dfd1aa0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808139 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021cd6000, {0x63c85b8, 0xc051ff9bf0}, 0xc02a7d7400, 0xc0a7126e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd6000, {0x63c85b8, 0xc051ff9bf0}, 0xc02a7d7400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a865aee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a865aee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc051ff9b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051ff9b60}, {0xc0054c53f8, 0x660089}, 0xc0054c5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc051ff9b60}, {0x7fe34e3f1c58}, 0xc02a7d7200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc051ff9ad0}, 0xc02a7d7200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}, 0xc071b572e0, 0xc0311097b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}, 0x3, 0xc074927280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0054c5980}}}, {0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}, 0xc071b572e0, 0xc074927280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc051ff9ad0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051ff9ad0}, {0x40, 0xc0054c5b70}, 0xc0054c5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc051ff9ad0}, {0x4e9d7a0, 0xc02a7d7200}, 0xc071b572e0, 0xc0311097b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc051ff9ad0}, 0xc07747f560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc078783680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc078783680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28062 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc0048da180, {0x63c85b8, 0xc02c8fba40}, 0x6, 0xc02385c720, 0xc000d3ab40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:269 +0x53f github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x63c85b8, 0xc02c8fba40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:222 +0x3c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801421 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc05154b5f0}, 0xc086c7a5b8, 0x1, 0xc04215f0a8, 0x519bbc, 0xc05a963020, 0xc0555da060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05154b5f0}, 0x120, 0xc04215f230, 0x5a962fc0, 0x0, 0x660069, 0xc05a963020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc05154b5f0}, 0xc05a962fc0, {{{{0x0, 0x0}, {0xc05bd28fc0, 0xe9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05154b5f0}, 0x6352410, 0x25eac168) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc05154b5f0}, {0xc05a962ea0, {0x174301101b0e9721, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc05154b5f0}, 0xc0a38f4fa0, 0xc04215f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05154b5f0}, 0x660089, {0xc05a962ea0, {0x174301101b0e9721, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc05154b5f0}, 0xc014ed1400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b0e9721, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b0e9721, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05154b560}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05154b560}, {0xc0421613f8, 0x660089}, 0xc042161400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05154b560}, {0x7fe32e752270}, 0xc014ed1200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05154b4d0}, 0xc014ed1200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}, 0xc040b123a0, 0xc0633f3968) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}, 0x3, 0xc053ce7a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042161980}}}, {0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}, 0xc040b123a0, 0xc053ce7a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05154b4d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05154b4d0}, {0x40, 0xc042161b70}, 0xc042161ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05154b4d0}, {0x4e9d7a0, 0xc014ed1200}, 0xc040b123a0, 0xc0633f3968) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05154b4d0}, 0xc046fa0c60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0484b6900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0484b6900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802575 [semacquire]: sync.runtime_SemacquireMutex(0x185a7d7, 0xc0, 0x7fe1fe66be20) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0x11869e5, 0x7fd6d354ba80, 0x7e73) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc09a23e4f8, 0x11a58d7, 0xc0253c44c0, 0x7fd6d354ba01) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7236e27, 0x3b81}, 0x0, 0xc04f5298f8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04f529680, {0xffffffffffffff15, 0xc09a23e5b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc04f529680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc04f529680, {0xc08f3d58c0, 0x32, 0x32}, 0x13, 0x15) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04f529680, {0xc050bcaf80, 0x32, 0x32}, {0xc08f3d58c0, 0x51c9860, 0xc08f3d58c0}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04f529680, {0xc050bcaf80, 0x32, 0x80}, {0xc08f3d58c0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07a958398, {0xc050bcaf80, 0x32, 0x80}, {0xc08f3d58c0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07a956cc0, {0xc08f3d58c0, 0x32, 0x80}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07a956cc0, {0xc050bcaf80, 0x32, 0xc08f3d58c0}, {0xc08f3d58c0, 0xc067132870, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07a956a00, {0xc08f3d58c0, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc056ec4da8, {{0xc067132870, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc035666700, {{0xc0344d24e0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0bbab5400, {0x63c85b8, 0xc018fa0fc0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc018fa0fc0}, {0x64a6560, 0xc035666700}, {0xc0344d24e0, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc018fa0fc0}, {0x6466ed8, 0xc056ec4b00}, {0xc0344d24e0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01944d908}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0b4406800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f21400, {0x63c85b8, 0xc018fa0fc0}, 0xc0b4406800, 0xc028112640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f21400, {0x63c85b8, 0xc018fa0fc0}, 0xc0b4406800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc018fa0f30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018fa0f30}, {0xc001c988f0, 0x660089}, 0xc001c988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc018fa0f30}, {0xc00102e4e0}, 0xc0b4406700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018fa0e40}, 0xc0b4406700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220b70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811380 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc060dcf380}, 0xc0672e9e90, 0x1, 0xc01a2050a8, 0x519bbc, 0xc01cbf14a0, 0xc04f001ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc060dcf380}, 0x120, 0xc01a205230, 0x1cbf1440, 0x0, 0x660069, 0xc01cbf14a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc060dcf380}, 0xc01cbf1440, {{{{0x0, 0x0}, {0xc05a3e4ea0, 0xe}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc060dcf380}, 0x6352410, 0x55bf3110) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc060dcf380}, {0xc012c3d9e0, {0x1743011076474074, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc060dcf380}, 0xc01cec4f00, 0xc01a2059f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc060dcf380}, 0x660089, {0xc012c3d9e0, {0x1743011076474074, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc060dcf380}, 0xc044a32300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0772dcc30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0772dcc30}, {0xc01a2073f8, 0x660089}, 0xc01a207400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0772dcc30}, {0xc04a25b460}, 0xc0be439600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0772dcba0}, 0xc0be439600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}, 0xc05ba789e0, 0xc0cdaf36f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}, 0x3, 0xc05279b2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a207980}}}, {0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}, 0xc05ba789e0, 0xc05279b2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0772dcba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0772dcba0}, {0x40, 0xc01a207b70}, 0xc01a207ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0772dcba0}, {0x4e9d7a0, 0xc0be439600}, 0xc05ba789e0, 0xc0cdaf36f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0772dcba0}, 0xc04f640de0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06dcb6480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06dcb6480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219792357 [semacquire]: sync.runtime_SemacquireMutex(0xc001ff8000, 0x3, 0xc001ff8000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074ab70) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc07dd5c358) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074ab60, 0x11869e5, 0x7fe3a2535000, 0x7edd, 0x7fdd526332c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc03db062a0, 0x7fe3a2535001, 0x7fe3a2535020, 0x2933, 0x2938) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc058e41340, {0xb3da71, 0x2933}, 0x0, 0xc0c182c978) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0c182c700, {0xffffffffffffff15, 0xc07dd5c5e0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0c182c700) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0c182c700, {0xc0e6a18540, 0x2b, 0x63}, 0xc057ee5ae0, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0c182c700, {0xc057ee5ae0, 0x2b, 0x9f}, {0xc0e6a18540, 0x2b, 0x63}, 0x1, 0x71) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0b1ef7040, {0xc057ee5ae0, 0x1, 0x63}, {0xc0e6a18540, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b1ef7040, {0xc057ee5ae0, 0x2b, 0x9f}, {0xc0e6a18540, 0x2b, 0x63}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b1ef5bc0, {0xc0e6a18540, 0x63, 0x9f}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b1ef5bc0, {0xc057ee5ae0, 0x2b, 0xc0e6a18540}, {0xc0e6a18540, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b1ef5900, {0xc0e6a18540, 0x2b, 0x63}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc063a31088, {{0xc02ec84c90, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc071724d00, {{0xc02ec84c90, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc01ecb9c00, {0x63c85b8, 0xc062f6f650}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062f6f650}, {0x64a6560, 0xc071724d00}, {0xc02ec84c90, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062f6f650}, {0x6466ed8, 0xc063a31080}, {0xc02ec84c90, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023238e58}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc07d06ef00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f82d000, {0x63c85b8, 0xc062f6f650}, 0xc07d06ef00, 0xc06f1a94a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f82d000, {0x63c85b8, 0xc062f6f650}, 0xc07d06ef00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062f6f5c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062f6f5c0}, {0xc0076c88f0, 0x660089}, 0xc0076c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062f6f5c0}, {0xc00102e4e0}, 0xc07d06ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062f6f560}, 0xc07d06ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04d8e2780}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790909 [runnable]: sync.runtime_SemacquireMutex(0xc07b916690, 0x8f, 0x7fdefc7d91c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc07b916538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc07b916560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0c59c4480, {0xffffffffffffff15, 0xc07b9166c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0c59c4480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0c59c4480, {0xc08f7bacf0, 0x32, 0x8c}, {0xc016927a40, 0x32, 0x93}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc05f5ad298, {0xc08f7bacf0, 0x32, 0x8c}, {0xc016927a40, 0x32, 0x93}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc05f5abbc0, {0xc016927a40, 0x93, 0x8c}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc05f5abbc0, {0xc08f7bacf0, 0x93, 0xc016927a40}, {0xc016927a40, 0xc0018fd050, 0x31}, 0x8c) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc05f5ab900, {0xc016927a40, 0x32, 0x93}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0cdbe3428, {{0xc0018fd050, 0x31, 0x8c}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07e0c5600, {{0xc023630b70, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0523c8800, {0x63c85b8, 0xc045a76630}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc045a76630}, {0x64a6560, 0xc07e0c5600}, {0xc023630b70, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc045a76630}, {0x6466ed8, 0xc0cdbe3180}, {0xc023630b70, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0bd09ac78}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0ac873f00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f6e1000, {0x63c85b8, 0xc045a76630}, 0xc0ac873f00, 0xc03183d0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f6e1000, {0x63c85b8, 0xc045a76630}, 0xc0ac873f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045a76570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045a76570}, {0xc0abc4a8f0, 0x660089}, 0xc0abc4a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045a76570}, {0xc00102e4e0}, 0xc0ac873e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045a76510}, 0xc0ac873e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc045a762d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 16385 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00d112b40) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 219801650 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0608518f0}, 0xc027c55560, 0x1, 0xc04cb370a8, 0x519bbc, 0xc03543e600, 0xc033c21aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0608518f0}, 0x1e0, 0xc04cb37230, 0x3543e5a0, 0x0, 0x660069, 0xc03543e600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0608518f0}, 0xc03543e5a0, {{{{0x0, 0x0}, {0xc0756ce000, 0x35}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0608518f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0608518f0}, {0xc02d4fb0e0, {0x1743011019b32a63, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0608518f0}, 0xc041b81720, 0xc04cb379f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0608518f0}, 0x660089, {0xc02d4fb0e0, {0x1743011019b32a63, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0608518f0}, 0xc063ac1700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019b32a63, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019b32a63, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060851860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060851860}, {0xc04cb393f8, 0x660089}, 0xc04cb39400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060851860}, {0x7fe33be604f8}, 0xc063ac1500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0608517d0}, 0xc063ac1500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}, 0xc019d67920, 0xc04daa51d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}, 0x3, 0xc06eaa7a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04cb39980}}}, {0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}, 0xc019d67920, 0xc06eaa7a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0608517d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0608517d0}, {0x40, 0xc04cb39b70}, 0xc04cb39ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0608517d0}, {0x4e9d7a0, 0xc063ac1500}, 0xc019d67920, 0xc04daa51d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0608517d0}, 0xc04f8043c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc064f07b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc064f07b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802553 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc023bd06c0}, 0xc0aebea600, 0xc0223825a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc023bd06c0}, 0xc0aebea600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024123859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024123859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067981fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067981fb0}, {0xc062c413f8, 0x660089}, 0xc062c41400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067981fb0}, {0x7fe338117418}, 0xc0aebea300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067981f20}, 0xc0aebea300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}, 0xc086683e00, 0xc01e0e12a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}, 0x3, 0xc036e3d0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062c41980}}}, {0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}, 0xc086683e00, 0xc036e3d0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc067981f20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067981f20}, {0x40, 0xc062c41b70}, 0xc062c41ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc067981f20}, {0x4e9d7a0, 0xc0aebea300}, 0xc086683e00, 0xc01e0e12a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc067981f20}, 0xc01c825020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0248bf0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0248bf0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28122 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0716f9400, {0x63c85b8, 0xc07221a870}, 0xc0708b1000, 0xc04e4cdb80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0716f9400, {0x63c85b8, 0xc07221a870}, 0xc0708b1000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107264d606, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107264d606, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07221a7e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07221a7e0}, {0xc07b53b4b0, 0x660089}, 0xc07b53b4b8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07221a7e0}, {0xc00102e4e0}, 0xc0708b0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07221a780}, 0xc0708b0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc011a614a0, {0x63c85b8, 0xc02ca0c060}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0x63c85b8, {0x63c85b8, 0xc02ca0c060}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec(0xc011a614a0, {0x63c85b8, 0xc02ca0c060}, 0xc07b53fa50) github.com/cockroachdb/cockroach/pkg/kv/txn.go:999 +0x10a github.com/cockroachdb/cockroach/pkg/kv.runTxn({0x63c85b8, 0xc02ca0c060}, 0xc01876a5a0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/db.go:948 +0x5a github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl(0x3e60f6a4d74f07c5, {0x63c85b8, 0xc02ca0c060}, 0x9b9d86b6, 0x12, 0x0) github.com/cockroachdb/cockroach/pkg/kv/db.go:911 +0x89 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:890 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).doFlushSingleStmtStats(0xc07b53fb48, {0x63c85b8, 0xc02ca0c060}, 0xedb7a8350, {0x0, 0xc07b53fb58, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/flush.go:175 +0x85 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).flushStmtStats.func1.1() github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/flush.go:83 +0x35 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).doFlush(0xc00cb72580, {0x63c85b8, 0xc02ca0c060}, 0xc07b53fc20, {0x4f7c2fe, 0x24}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/flush.go:123 +0x115 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).flushStmtStats.func1({0x63c85b8, 0xc02ca0c060}, 0x498a700) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/flush.go:82 +0xa9 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).IterateStatementStats(0xc0ade63cd0, {0x63c85b8, 0xc02ca0c060}, 0xedb7a7760, 0xc07b53fce0) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:126 +0x66 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).flushStmtStats(0xc00cb72580, {0x63c85b8, 0xc02ca0c060}, {0x63c85b8, 0xc02ca0c060, 0x0}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/flush.go:80 +0xc5 github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).Flush(0xc00cb72580, {0x63c85b8, 0xc02ca0c060}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/flush.go:73 +0x3fa github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1({0x63c85b8, 0xc02ca0c060}) github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/provider.go:152 +0x299 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810123 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035a40, {0x63c85b8, 0xc08822b350}, 0xc0569ac318, 0x1, 0xc0106b50a8, 0x519bbc, 0xc0532b8240, 0xc093732c00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc064b67680, {0x63c85b8, 0xc08822b350}, 0xc064b676b8, 0xc0106b51a8, 0x41, 0x60, 0xffffffffffffffff, 0xc0532b8240, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035a40, {0x63c85b8, 0xc08822b350}, 0xc0532b8000, {{{{0xc084427560, 0x1}, {0xc0477b58c0, 0x1}}, {{0xc05261ed80, 0x5}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08822b350}, 0x63521a0, 0x50b34328) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd8b0, {0x63c85b8, 0xc08822b350}, {0xc06acde360, {0x174301100a5095a2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd8b0, {0x63c85b8, 0xc08822b350}, 0xc04c449ae0, 0xc0106b59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08822b350}, 0x660089, {0xc06acde360, {0x174301100a5095a2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e096000, {0x63c85b8, 0xc08822b350}, 0xc051d52c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100a5095a2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100a5095a2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08822b2c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08822b2c0}, {0xc0106b73f8, 0x660089}, 0xc0106b7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08822b2c0}, {0x0}, 0xc051d52a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08822b1d0}, 0xc051d52a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}, 0xc0198db3a0, 0xc0462b5668) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}, 0x3, 0xc074747640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0106b7980}}}, {0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}, 0xc0198db3a0, 0xc074747640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08822b1d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08822b1d0}, {0x40, 0xc0106b7b70}, 0xc0106b7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08822b1d0}, {0x4e9d7a0, 0xc051d52a00}, 0xc0198db3a0, 0xc0462b5668) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08822b1d0}, 0xc02daff500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06acde240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06acde240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 214445160 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaffb3fb0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06ef08580, 0xc0279b9000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06ef08580, {0xc0279b9000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06ef08580, {0xc0279b9000, 0x735, 0xc0636f72c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03abf79d8, {0xc0279b9000, 0x7eda79, 0xc0279b9005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0847c28a0, {0xc0279b9000, 0xc084f32c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0731137b8, {0xc0279b9000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06f448cf8, {0x631b900, 0xc0731137b8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06f448a80, {0x7fe42ab59310, 0xc0847c28a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06f448a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06f448a80, {0xc04dc67000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc067b683c0, {0xc04dc67000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05457eb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05457eb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05457f0e0, {0x634aac0, 0xc05457eb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05457ea80, 0xc07b30ede0, 0xc076ecf42b, 0xc076ecf42d, {0x635bec0, 0xc0719a1290}, 0xc000d6ae00, 0xc01aaed818, 0xc076ecf429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05457ea80, {0x63c85b8, 0xc0176d6720}, 0xc07b30edd0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc00a329009, 0x8}}, 0x0, 0xc0176d66c0, 0xc0176d66f0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0176d6720}, {0x643d810, 0xc0847c28a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07237c390}, {0x643d810, 0xc0847c28a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07237c390}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810252 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bc70000, {0x63c85b8, 0xc068870ff0}, 0xc06ecec800, 0xc0d43937c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bc70000, {0x63c85b8, 0xc068870ff0}, 0xc06ecec800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cab57e2, 0x0, 0x0}, 0xc03fc6f230, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cab57e2, 0x0, 0x0}, 0xc03fc6f230, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc068870f60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc068870f60}, {0xc0272a53f8, 0x660089}, 0xc0272a5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc068870f60}, {0x7fe3b8b316d0}, 0xc06ecec600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc068870ed0}, 0xc06ecec600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}, 0xc060cbc760, 0xc028878258) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}, 0x3, 0xc01d4b5680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0272a5980}}}, {0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}, 0xc060cbc760, 0xc01d4b5680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc068870ed0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc068870ed0}, {0x40, 0xc0272a5b70}, 0xc0272a5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc068870ed0}, {0x4e9d7a0, 0xc06ecec600}, 0xc060cbc760, 0xc028878258) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc068870ed0}, 0xc039116d20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0616e39e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0616e39e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219784362 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8510, 0xc06d225b40}, 0x12, 0xc075037700) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc03bfeef20, {0x63c8510, 0xc06d225b40}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/txn.go:696 +0x74 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc06cdefc28, {0x63c8510, 0xc06d225b40}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:225 +0xb6 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc06cdefc00, {{0x63c8510, 0xc06d225b40}, 0xc003f2f500, 0xc066e252f0}) github.com/cockroachdb/cockroach/pkg/sql/update.go:196 +0x210 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc056575a10, {{0x63c8510, 0xc06d225b40}, 0xc003f2f500, 0xc066e252f0}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc0889c5930, 0x63cb098}, {0x63caa08, 0xc056575a10}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:516 +0x147 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0889c5930, {0x63caa08, 0xc056575a10}, {0x4ed688c, 0x5}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c9 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0889c5930, {0x63caa08, 0xc056575a10}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x63c8510, 0xc06d225b40}, 0xc003f2f500, 0xc066e252f0}, {0x63caa08, 0xc056575a10}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:519 +0x125 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc035c82e00, {0x63c8510, 0xc06d225b40}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:146 +0x87 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:120 +0x34 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x200000000) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc038bb9320, {0x63c8510, 0xc06d225b40}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:119 +0x6b github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc038bb9320, {0x63c8510, 0xc06d225b40}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:753 +0x3a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc06fbbdba0, {0x63c8510, 0xc06d225b40}, 0xc038bb9320) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:469 +0x258 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc066e255e8, {0x63c8510, 0xc06d225b40}, 0xc03bfeef20) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:303 +0x205 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc06d225780}, 0xc06c7069a0, 0xc03bfeef20, 0xc01e834d00, 0xc0686b6000, 0xc066e255e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc06d225780}, 0xc066e252f0, 0xc06c7069a0, 0xc068383140, {{0x63caa08, 0xc056575a10}, 0x0}, 0xc0686b6000) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc066e24c00, {0x63c8510, 0xc06d225780}, 0xc066e252f0, 0xc06d225780, {0x7fe42b790cf0, 0xc068383140}, 0x0, 0xc0672f35b8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc066e24c00, {0x63c8510, 0xc06d225780}, 0xc066e252f0, {0x7fe42b790cf0, 0xc068383140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc066e24c00, {0x63c85b8, 0xc04fec82a0}, {{0x63f9d78, 0xc06d218080}, {0xc060dc2a31, 0x83}, 0x6, 0x1}, 0xc06047ec60, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc066e24c00, {0x63c85b8, 0xc04fec82a0}, {{0x63f9d78, 0xc06d218080}, {0xc060dc2a31, 0x83}, 0x6, 0x1}, 0xc06047ec60, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc066e24c00, {0x63c85b8, 0xc04fec82a0}, {0xc06047ec60, {0xc0672f34a0, 0x6, 0x6}, {0x9bf64bc, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0a2a38698, 0x0}, 0x0, {0x1a16cfe8, 0xedb7a834f, 0x0}, 0x1}, 0xc066e24c00, 0xc098dad980, 0xc098dad970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc066e24c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc066e24c00, {0x63c8510, 0xc07684e640}, 0xc075de3dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0768387a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc07684e640}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0768387a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812795 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794930 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc03e235a40}, 0xc06ef68cb0, {0x646cc48, 0xc03e235aa0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc03e235a40}, 0xedb7a834e, 0xc04c8dd680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc03e235a40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07b8c64e0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc06ef68ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc07b8c64e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28587 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 219662287 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc0133782d0, {0x63c85b8, 0xc060cf2cc0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc0133782d0, {0x63c85b8, 0xc060cf2cc0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc03c0dc230, {0x63c85b8, 0xc060cf2cc0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc066618000, {0x63c85b8, 0xc060cf2cc0}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc08b2f4f20) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:264 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc08b2fe3c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc08abf5860) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:266 +0x73 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:291 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc057798d40) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc08abf5860) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*noopProcessor).Next(0xc08b2f7800) github.com/cockroachdb/cockroach/pkg/sql/rowexec/noop.go:74 +0x46 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc08b7fe000) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:210 +0x130 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc08abf5a40) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:266 +0x73 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:291 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x63c8510) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc08abf5a40) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).next(0xc06053aea0) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:128 +0x3b github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:145 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x0) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Next(0xc06053aea0) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:150 +0x3e github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run({0x63c8510, 0xc08b2fe340}, {0x63f18f8, 0xc06053aea0}, {0x6343d80, 0xc03fea4e00}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:187 +0x55 github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc06053aea0, {0x63c8510, 0xc08b2fe340}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:754 +0x5b github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc08b2ead00, {0x63c8510, 0xc08b2fe340}, 0xc06053aea0) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:469 +0x258 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc076f3b5e8, {0x63c8510, 0xc08b2fe340}, 0xc045dd8000) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:303 +0x205 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc08b2fe140}, 0xc08b7b67e0, 0xc045dd8000, 0xc08abefd80, 0xc03fea4e00, 0xc076f3b5e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc08b2fe140}, 0xc076f3b2f0, 0xc08b7b67e0, 0xc06ca27890, {{0x63ca960, 0xc04c1c4fa0}, 0x0}, 0xc03fea4e00) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc076f3ac00, {0x63c8510, 0xc08b2fe140}, 0xc076f3b2f0, 0xc08b2fe140, {0x7fe3e0f340e8, 0xc06ca27890}, 0x0, 0xc02a0ac718) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc076f3ac00, {0x63c8510, 0xc08b2fe140}, 0xc076f3b2f0, {0x7fe3e0f340e8, 0xc06ca27890}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc076f3ac00, {0x63c85b8, 0xc06ca26900}, {{0x63f8d78, 0xc017be9220}, {0xc09ae5b0dc, 0x155}, 0x0, 0x1}, 0xc08b2c1c20, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc076f3ac00, {0x63c85b8, 0xc06ca26900}, {{0x63f8d78, 0xc017be9220}, {0xc09ae5b0dc, 0x155}, 0x0, 0x1}, 0xc08b2c1c20, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc076f3ac00, {0x63c85b8, 0xc06ca26900}, {0xc08b2c1c20, {0x9bf6f48, 0x0, 0x0}, {0xc056fca9a0, 0xe, 0xe}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc09ae5b244, 0x3}, 0x3e8, {0x2116f9fd, 0xedb7a833f, 0x0}, 0x1}, 0xc076f3ac00, 0xc07c8c5980, 0xc07c8c5970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc076f3ac00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc076f3ac00, {0x63c8510, 0xc034eca5c0}, 0xc056adbdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04189deb0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc034eca5c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04189deb0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 30499 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc01c9e8180, {0x63c85b8, 0xc0318a0e70}, 0x10, 0xc02df738c0, 0xc000d3ab40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:269 +0x53f github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x63c85b8, 0xc0318a0e70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:222 +0x3c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802122 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ecd5800, {0x63c85b8, 0xc05454ede0}, 0xc070038400, 0xc0a8d6f720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ecd5800, {0x63c85b8, 0xc05454ede0}, 0xc070038400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020b053cb, 0x0, 0x0}, 0xc024e283e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020b053cb, 0x0, 0x0}, 0xc024e283e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05454ecf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05454ecf0}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05454ecf0}, {0xc00102e4e0}, 0xc070038200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05454ec30}, 0xc070038200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc022b7e480}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc05454eba0}, {0xc00dfd36b0, 0x24}, 0x46c7c6f10, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x4c1af3}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc05454eba0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806936 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b9a2000, {0x63c85b8, 0xc091c80420}, 0xc05fed60a8, 0x1, 0xc004b370a8, 0x519bbc, 0xc03631ca80, 0xc088353b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc091c80420}, 0x120, 0xc004b37230, 0x3631ca20, 0x0, 0x660069, 0xc03631ca80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b9a2000, {0x63c85b8, 0xc091c80420}, 0xc03631ca20, {{{{0x0, 0x0}, {0xc09fae0d80, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc091c80420}, 0x6352410, 0x6618b8d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0fb3b0, {0x63c85b8, 0xc091c80420}, {0xc03631c900, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0fb3b0, {0x63c85b8, 0xc091c80420}, 0xc072aecc80, 0xc004b379f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc091c80420}, 0x660089, {0xc03631c900, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b996400, {0x63c85b8, 0xc091c80420}, 0xc097d85300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc091c80390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc091c80390}, {0xc004b393f8, 0x660089}, 0xc004b39400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc091c80390}, {0x3fefffdc5f52ef1d}, 0xc097d85100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc091c80300}, 0xc097d85100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}, 0xc08623fb00, 0xc05fed7c20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}, 0x3, 0xc08ec2d5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004b39980}}}, {0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}, 0xc08623fb00, 0xc08ec2d5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc091c80300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc091c80300}, {0x40, 0xc004b39b70}, 0xc004b39ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc091c80300}, {0x4e9d7a0, 0xc097d85100}, 0xc08623fb00, 0xc05fed7c20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc091c80300}, 0xc0659e1620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590b86c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590b86c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219793069 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0ce8d8f60}, 0xc007773cb0, {0x646cc48, 0xc0ce8d8fc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0ce8d8f60}, 0xedb7a834e, 0xc0608485a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0ce8d8f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02a28bbc0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc007773ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc02a28bbc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 15950 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc0105cf680, {0x63c85b8, 0xc0232f0390}, 0x73f6, 0xc018b009d8, 0xc000d3ab40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:269 +0x53f github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x63c85b8, 0xc0232f0390}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:222 +0x3c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807005 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc01cb2cf60}, {0xc057291600, 0x4, 0xc09c0c0e58}, {0x0, {0x1743010ff4bb2979, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc01cb2cf60}, 0x1871d, 0xc043cfe240, 0x20, 0xc07a2d77a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc01cb2cf60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808973 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc03d635800}, 0xc015376c48, 0x1, 0xc01a3870a8, 0x519bbc, 0xc04e3f4720, 0xc060ca6600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03d635800}, 0x120, 0xc01a387230, 0x4e3f46c0, 0x0, 0x660069, 0xc04e3f4720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc03d635800}, 0xc04e3f46c0, {{{{0x0, 0x0}, {0xc02773d8c0, 0x126}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03d635800}, 0x6352410, 0x29b7a6a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc03d635800}, {0xc04e3f45a0, {0x174301105e6e348b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc03d635800}, 0xc0314e17c0, 0xc01a3879f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03d635800}, 0x660089, {0xc04e3f45a0, {0x174301105e6e348b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc03d635800}, 0xc056dd0100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e6e348b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e6e348b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03d635620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d635620}, {0xc01a3893f8, 0x660089}, 0xc01a389400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03d635620}, {0x7fdd4a340900}, 0xc008453f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03d635590}, 0xc008453f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}, 0xc0746ffb60, 0xc056516840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}, 0x3, 0xc06249fe40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a389980}}}, {0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}, 0xc0746ffb60, 0xc06249fe40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03d635590}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d635590}, {0x40, 0xc01a389b70}, 0xc01a389ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03d635590}, {0x4e9d7a0, 0xc008453f00}, 0xc0746ffb60, 0xc056516840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03d635590}, 0xc038a08600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04e3f4480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04e3f4480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28615 [runnable]: sync.runtime_notifyListWait(0xc0003df278, 0xfc616) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xadd280) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc0003df260, {0x63c8510, 0xc00b49c0c0}, 0x327c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xde github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x63c85b8, 0xc02cc73500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x357 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810133 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc076482000}, {0xc0aedd0420, 0x2, 0xc09c0c6658}, {0x0, {0x1743010ff624ed79, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc076482000}, 0x18701, 0xc09c29e120, 0x20, 0xc020362080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc076482000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801054 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f6c7800, {0x63c85b8, 0xc063310c90}, 0xc03fdb9300, 0xc04bf3bf40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f6c7800, {0x63c85b8, 0xc063310c90}, 0xc03fdb9300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011018bca093, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011018bca093, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc063310c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063310c00}, {0xc0129f93f8, 0x660089}, 0xc0129f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc063310c00}, {0x33}, 0xc03fdb8f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc063310b70}, 0xc03fdb8f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}, 0xc06c54c1c0, 0xc0210f6060) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}, 0x3, 0xc03b9d1340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0129f9980}}}, {0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}, 0xc06c54c1c0, 0xc03b9d1340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc063310b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063310b70}, {0x40, 0xc0129f9b70}, 0xc0129f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc063310b70}, {0x4e9d7a0, 0xc03fdb8f00}, 0xc06c54c1c0, 0xc0210f6060) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc063310b70}, 0xc050933680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04b1eed80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04b1eed80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812711 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018564a00, {0x63c85b8, 0xc02557bef0}, 0xc0269f5dd0, 0x1, 0xc03d83b0a8, 0x519bbc, 0xc099587000, 0xc0441b2a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04667dd40, {0x63c85b8, 0xc02557bef0}, 0xc04667dd88, 0xc03d83b1a8, 0x30, 0x40, 0xffffffffffffffff, 0xc099587000, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018564a00, {0x63c85b8, 0xc02557bef0}, 0xc03df19bc0, {{{{0x0, 0x0}, {0xc0764d5200, 0x1}}, {{0xc04e62e900, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02557bef0}, 0x63521a0, 0x42013470) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0185325f0, {0x63c85b8, 0xc02557bef0}, {0xc02a23a6c0, {0x174301101623c7df, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0185325f0, {0x63c85b8, 0xc02557bef0}, 0xc086299c20, 0xc03d83b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02557bef0}, 0x660089, {0xc02a23a6c0, {0x174301101623c7df, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018574800, {0x63c85b8, 0xc02557bef0}, 0xc03486e700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101623c7df, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101623c7df, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02557be30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02557be30}, {0xc03d83d3f8, 0x660089}, 0xc03d83d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02557be30}, {0x7fe426bc7510}, 0xc03486e300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02557bd70}, 0xc03486e300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}, 0xc00abd2a80, 0xc047919a10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}, 0x3, 0xc01352b240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03d83d980}}}, {0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}, 0xc00abd2a80, 0xc01352b240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02557bd70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02557bd70}, {0x40, 0xc03d83db70}, 0xc03d83dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02557bd70}, {0x4e9d7a0, 0xc03486e300}, 0xc00abd2a80, 0xc047919a10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02557bd70}, 0xc03df19a40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e9d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e9d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28701 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc02cdd8a50, {0xc018c97ae0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc02cdd8a50, {0xc018c97ae0, 0xc01f841cf8, 0xc007b58a80}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0215c36e0, {0xc018c97ae0, 0xc007b58af8, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc0215c36e0}, {0xc018c97ae0, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc013e66ea0, {0xc018c97ae0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc018c97ad0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x58, 0xc013e66ea0, {0x0, 0x0}, 0x7fffffff, 0xc007b58da8, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0xc04d5f6360, {0x0, 0x0}, {0x4cfd920, 0xc04d5f6360}, 0x203013, 0xc007b58da8, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc00b755110, {0x4cfd920, 0xc04d5f6360}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb.(*sideTransportPushUpdatesServer).Recv(0xc00a5f43c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:365 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run.func1({0x63c85b8, 0xc0215c3b30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:304 +0xc2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810258 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b26c000, {0x63c85b8, 0xc01d33a060}, 0xc062d7ba00, 0xc02f38fcc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc01d33a060}, 0xc062d7ba00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c0cb3e9, 0x0, 0x0}, 0xc03cc768f0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c0cb3e9, 0x0, 0x0}, 0xc03cc768f0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc061235fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc061235fb0}, {0xc0084893f8, 0x660089}, 0xc008489400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc061235fb0}, {0x7fe3ba654988}, 0xc062d7b700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc061235ef0}, 0xc062d7b700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}, 0xc02ff823a0, 0xc041bf3c20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}, 0x3, 0xc05095fe40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc008489980}}}, {0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}, 0xc02ff823a0, 0xc05095fe40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc061235ef0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc061235ef0}, {0x40, 0xc008489b70}, 0xc008489ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc061235ef0}, {0x4e9d7a0, 0xc062d7b700}, 0xc02ff823a0, 0xc041bf3c20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc061235ef0}, 0xc0aadfac60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc073564ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc073564ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807708 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ff75c00, {0x63c85b8, 0xc04f58e030}, 0xc04c88ea00, 0xc083df4280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75c00, {0x63c85b8, 0xc04f58e030}, 0xc04c88ea00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011059160f37, 0x0, 0x0}, 0xc09215ccb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011059160f37, 0x0, 0x0}, 0xc09215ccb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023113f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023113f80}, {0xc00753b3f8, 0x660089}, 0xc00753b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023113f80}, {0x40e8faff2ddd4c3a}, 0xc04c88e600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc023113e90}, 0xc04c88e600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}, 0xc0779f4f80, 0xc03f10fd88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}, 0x3, 0xc04f30f700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00753b980}}}, {0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}, 0xc0779f4f80, 0xc04f30f700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc023113e90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023113e90}, {0x40, 0xc00753bb70}, 0xc00753bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc023113e90}, {0x4e9d7a0, 0xc04c88e600}, 0xc0779f4f80, 0xc03f10fd88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc023113e90}, 0xc06e7144e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc022e510e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc022e510e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219792994 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc046d3a7e0}, 0xc01ad20cb0, {0x646cc48, 0xc046d3a840}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc046d3a7e0}, 0xedb7a834e, 0xc063b53320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc046d3a7e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07d21ee40}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01ad20ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc07d21ee40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811864 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edab040, {0x63c85b8, 0xc0ab06c060}, 0xc072326ff0, 0x1, 0xc0062bd0a8, 0x519bbc, 0xc073207320, 0xc0187cf320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0ab06c060}, 0x1e0, 0xc0062bd230, 0x732072c0, 0x0, 0x660069, 0xc073207320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edab040, {0x63c85b8, 0xc0ab06c060}, 0xc0732072c0, {{{{0x0, 0x0}, {0xc0aca898c0, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc04e157b00, {0x63c85b8, 0xc0ab06c060}, 0xc05fb1f690, 0x62bd428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e7017c0, {0x63c85b8, 0xc0ab06c060}, {0xc064e3f680, {0x1743011065ea6c77, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e7017c0, {0x63c85b8, 0xc0ab06c060}, 0xc0cb6e5900, 0xc0062bd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0ab06c060}, 0x660089, {0xc064e3f680, {0x1743011065ea6c77, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ef1c000, {0x63c85b8, 0xc0ab06c060}, 0xc059d4dd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ea6c77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ea6c77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045b33fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045b33fb0}, {0xc0062bf3f8, 0x660089}, 0xc0062bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045b33fb0}, {0x7fe429f4a0a0}, 0xc059d4db00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045b33f20}, 0xc059d4db00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}, 0xc00a498e60, 0xc072327b30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}, 0x3, 0xc0797e93c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0062bf980}}}, {0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}, 0xc00a498e60, 0xc0797e93c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045b33f20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045b33f20}, {0x40, 0xc0062bfb70}, 0xc0062bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045b33f20}, {0x4e9d7a0, 0xc059d4db00}, 0xc00a498e60, 0xc072327b30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045b33f20}, 0xc0505f5260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc059511e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc059511e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28503 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1({0x63c8510, 0xc017d65800}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:116 +0x15a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219803828 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01e7d3000, {0x63c85b8, 0xc0543dfc20}, 0xc02f98a000, 0xc0496fd400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e7d3000, {0x63c85b8, 0xc0543dfc20}, 0xc02f98a000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103216848b, 0x0, 0x0}, 0xc0530eaf80, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103216848b, 0x0, 0x0}, 0xc0530eaf80, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0543dfb90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0543dfb90}, {0xc0624453f8, 0x660089}, 0xc062445400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0543dfb90}, {0x0}, 0xc060de7e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0543dfb00}, 0xc060de7e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}, 0xc055026940, 0xc01f5b8bb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}, 0x3, 0xc041e20a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062445980}}}, {0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}, 0xc055026940, 0xc041e20a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0543dfb00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0543dfb00}, {0x40, 0xc062445b70}, 0xc062445ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0543dfb00}, {0x4e9d7a0, 0xc060de7e00}, 0xc055026940, 0xc01f5b8bb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0543dfb00}, 0xc073e9aa80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28400 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0022607e0, {0x63c8510, 0xc0203a8c80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:310 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0022607e0, {0x63c85b8, 0xc01c227590}, 0x5) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:335 +0xbb github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x63c85b8, 0xc01c227590}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790218 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0466a6a80}, 0xc027909cb0, {0x646cc48, 0xc0466a6b40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0466a6a80}, 0xedb7a834e, 0xc07d458240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0466a6a80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc047adf0b0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc027909ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc047adf0b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806846 [IO wait]: internal/poll.runtime_pollWait(0x7fdb124bc7c0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0748d3400, 0xc08f339000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0748d3400, {0xc08f339000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0748d3400, {0xc08f339000, 0x735, 0xc05c5ae8c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc063fdd6b8, {0xc08f339000, 0x7eda79, 0xc08f339005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0543102a0, {0xc08f339000, 0xc004eb8c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a90420, {0xc08f339000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc05ca513f8, {0x631b900, 0xc043a90420}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc05ca51180, {0x7fe42ab59310, 0xc0543102a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc05ca51180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc05ca51180, {0xc08a17d000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0234b8420, {0xc08a17d000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0849c6088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0849c6088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0849c6660, {0x634aac0, 0xc0849c6088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0849c6000, 0xc054daa970, 0xc004eb942b, 0xc004eb942d, {0x635bec0, 0xc012662bd0}, 0xc000d6ae00, 0xc08d6df014, 0xc004eb9429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0849c6000, {0x63c85b8, 0xc09323d710}, 0xc054daa960, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0446ec03b, 0x8}}, 0x0, 0xc09323d6b0, 0xc09323d6e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc09323d710}, {0x643d810, 0xc0543102a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc09323c720}, {0x643d810, 0xc0543102a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc09323c720}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809513 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ba0ea00, {0x63c85b8, 0xc018279e30}, 0xc02d10cc00, 0x1, 0xc0275bf0a8, 0x519bbc, 0xc0838a3b60, 0xc05330bc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc018279e30}, 0x120, 0xc0275bf230, 0x838a3b00, 0x0, 0x660069, 0xc0838a3b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ba0ea00, {0x63c85b8, 0xc018279e30}, 0xc0838a3b00, {{{{0x0, 0x0}, {0xc01f392360, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc018279e30}, 0x6352410, 0x450cb678) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b9f8870, {0x63c85b8, 0xc018279e30}, {0xc0838a39e0, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b9f8870, {0x63c85b8, 0xc018279e30}, 0xc0568e3ae0, 0xc0275bf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc018279e30}, 0x660089, {0xc0838a39e0, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ba1d000, {0x63c85b8, 0xc018279e30}, 0xc05ef03000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc018279b90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018279b90}, {0xc0275c13f8, 0x660089}, 0xc0275c1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc018279b90}, {0x3fefffff57ae8673}, 0xc05ef02e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018279830}, 0xc05ef02e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}, 0xc036c65860, 0xc038b121c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}, 0x3, 0xc0bdb96480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0275c1980}}}, {0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}, 0xc036c65860, 0xc0bdb96480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc018279830}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018279830}, {0x40, 0xc0275c1b70}, 0xc0275c1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc018279830}, {0x4e9d7a0, 0xc05ef02e00}, 0xc036c65860, 0xc038b121c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc018279830}, 0xc04e1565a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0838a38c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0838a38c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219640620 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff35850, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc011576f00, 0xc041307000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc011576f00, {0xc041307000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc011576f00, {0xc041307000, 0x735, 0xc05650b900}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0534f3840, {0xc041307000, 0x7eda79, 0xc041307005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01c5600c0, {0xc041307000, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02d7d3950, {0xc041307000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0a6b97778, {0x631b900, 0xc02d7d3950}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0a6b97500, {0x7fe42ab59310, 0xc01c5600c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0a6b97500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0a6b97500, {0xc0f0126000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc026fa9158, {0xc0f0126000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0986d4088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0986d4088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0986d4660, {0x634aac0, 0xc0986d4088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0986d4000, 0xc02ef7c910, 0xc07a70b42b, 0xc07a70b42d, {0x635bec0, 0xc051b107e0}, 0xc000d6ae00, 0xc054ba3558, 0xc07a70b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0986d4000, {0x63c85b8, 0xc06ecf2090}, 0xc02ef7c900, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc038cd8047, 0x8}}, 0x0, 0xc06ecf2030, 0xc06ecf2060, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06ecf2090}, {0x643d810, 0xc01c5600c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06c3fc540}, {0x643d810, 0xc01c5600c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06c3fc540}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219774421 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3df6e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc08c54aa80, 0xc019f7f800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc08c54aa80, {0xc019f7f800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc08c54aa80, {0xc019f7f800, 0x735, 0xc055177180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc065bec738, {0xc019f7f800, 0x7eda79, 0xc019f7f805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0388b1c20, {0xc019f7f800, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0731135a8, {0xc019f7f800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04c030cf8, {0x631b900, 0xc0731135a8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04c030a80, {0x7fe42ab59310, 0xc0388b1c20}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04c030a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04c030a80, {0xc06f388000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06d725e60, {0xc06f388000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03d371588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03d371588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03d371b60, {0x634aac0, 0xc03d371588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03d371500, 0xc02d882420, 0xc08304742b, 0xc08304742d, {0x635bec0, 0xc04f023830}, 0xc000d6ae00, 0xc0430ce734, 0xc083047429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03d371500, {0x63c85b8, 0xc03ddf3ad0}, 0xc02d882410, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc083453038, 0x8}}, 0x0, 0xc03ddf3a70, 0xc03ddf3aa0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03ddf3ad0}, {0x643d810, 0xc0388b1c20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0676fcff0}, {0x643d810, 0xc0388b1c20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0676fcff0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790060 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99a58, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc005cb5700, 0xc07c066000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc005cb5700, {0xc07c066000, 0x35a6, 0x35a6}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc005cb5700, {0xc07c066000, 0x35a6, 0xc0639efb80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03102fb40, {0xc07c066000, 0x7eda79, 0xc07c066005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06b0f5620, {0xc07c066000, 0x0, 0x35a6}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02532d560, {0xc07c066000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0525ed778, {0x631b900, 0xc02532d560}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0525ed500, {0x7fe42ab59310, 0xc06b0f5620}, 0x7fe41fe99a98) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0525ed500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0525ed500, {0xc08e00e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0462b4eb8, {0xc08e00e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0ccc84088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0ccc84088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0ccc84660, {0x634aac0, 0xc0ccc84088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0ccc84000, 0xc07092f550, 0xc041bed42b, 0xc041bed42d, {0x635bec0, 0xc07724e2d0}, 0xc000d6ae00, 0xc0515bd798, 0xc041bed429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0ccc84000, {0x63c85b8, 0xc026c9b4d0}, 0xc07092f540, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc078a4c037, 0x8}}, 0x0, 0xc026c9b470, 0xc026c9b4a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc026c9b4d0}, {0x643d810, 0xc06b0f5620}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04608e7e0}, {0x643d810, 0xc06b0f5620}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04608e7e0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801336 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bab4000, {0x63c85b8, 0xc04e2183c0}, 0xc043476200, 0xc075c2c280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bab4000, {0x63c85b8, 0xc04e2183c0}, 0xc043476200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ac7b078, 0x0, 0x0}, 0xc06516fac0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ac7b078, 0x0, 0x0}, 0xc06516fac0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e218330}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e218330}, {0xc04d3a73f8, 0x660089}, 0xc04d3a7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e218330}, {0x413134343480f0fa}, 0xc043476000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e2182a0}, 0xc043476000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}, 0xc0682f6fa0, 0xc076dbe840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}, 0x3, 0xc0d4d03d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04d3a7980}}}, {0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}, 0xc0682f6fa0, 0xc0d4d03d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e2182a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e2182a0}, {0x40, 0xc04d3a7b70}, 0xc04d3a7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e2182a0}, {0x4e9d7a0, 0xc043476000}, 0xc0682f6fa0, 0xc076dbe840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e2182a0}, 0xc0c2e6faa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ab856000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ab856000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812224 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0185643c0, {0x63c85b8, 0xc045fa7890}, 0xc0c11de9d8, 0x1, 0xc03e7c50a8, 0x519bbc, 0xc00cf3b920, 0xc0c9976e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc075effb00, {0x63c85b8, 0xc045fa7890}, 0xc075effb30, 0xc03e7c51a8, 0x16, 0x18, 0xffffffffffffffff, 0xc00cf3b920, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0185643c0, {0x63c85b8, 0xc045fa7890}, 0xc00cf3b8c0, {{{{0xc0d10eab40, 0x1}, {0xc04c615320, 0xf}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc045fa7890}, 0x6352230, 0x204d31a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0185322d0, {0x63c85b8, 0xc045fa7890}, {0xc00cf3b7a0, {0x174301108514419e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0185322d0, {0x63c85b8, 0xc045fa7890}, 0xc0650bea00, 0xc03e7c59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc045fa7890}, 0x660089, {0xc00cf3b7a0, {0x174301108514419e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018570c00, {0x63c85b8, 0xc045fa7890}, 0xc06bbd4900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108514419e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108514419e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045fa7800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045fa7800}, {0xc03e7c73f8, 0x660089}, 0xc03e7c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045fa7800}, {0x0}, 0xc06bbd4700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045fa7770}, 0xc06bbd4700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}, 0xc0ccf7f1e0, 0xc038160e10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}, 0x3, 0xc049737a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03e7c7980}}}, {0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}, 0xc0ccf7f1e0, 0xc049737a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045fa7770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045fa7770}, {0x40, 0xc03e7c7b70}, 0xc03e7c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045fa7770}, {0x4e9d7a0, 0xc06bbd4700}, 0xc0ccf7f1e0, 0xc038160e10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045fa7770}, 0xc021af07e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc050cdca20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc050cdca20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28502 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00141adc0, {0x63c8510, 0xc017d657c0}, 0xc02c253b30, 0xc01e877a40) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:331 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x63c8510, 0xc017d657c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:288 +0x38 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 218566377 [semacquire, 2 minutes]: sync.runtime_Semacquire(0x48) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc000d3ab40) GOROOT/src/sync/waitgroup.go:130 +0x71 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RunConsistencyCheck(0xc01860dc00, {0x63c8580, 0xc04fc71200}, {{{0xc018636840, 0x38, 0x40}, {0x0, 0x0, 0x0}, 0x0}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_consistency.go:424 +0x6fa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).CheckConsistency(0xc01860dc00, {0x63c8580, 0xc04fc71200}, {{{0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_consistency.go:107 +0x1b4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*consistencyQueue).process(0xc00d545230, {0x63c8580, 0xc04fc71200}, 0xc01860dc00, {0xc04fc71200, 0x0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/consistency_queue.go:199 +0x193 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica.func1({0x63c8580, 0xc04fc71200}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:977 +0x26e github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02762b4a0}, {0xc04be7fb90, 0x2d}, 0x29e8d60800, 0xc01fad5e28) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica(0xc00d507180, {0x63c85b8, 0xc07df7bc50}, {0x6463f50, 0xc01860dc00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:936 +0x3f5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2.1({0x63c85b8, 0xc07df7bc50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:855 +0xde github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811981 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc023d6f400, {0x63c85b8, 0xc0828ccd20}, 0xc0cc964420, 0x1, 0x0, 0x1, 0xc03be58a80, 0xc0624e3020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0828ccd20}, 0x120, 0xc0364ca728, 0x3be58a20, 0x0, 0x660069, 0xc03be58a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc023d6f400, {0x63c85b8, 0xc0828ccd20}, 0xc03be58a20, {{{{0x0, 0x0}, {0xc0642746c0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0828ccd20}, 0x6352410, 0x204d2360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc023ce53b0, {0x63c85b8, 0xc0828ccd20}, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc023ce53b0, {0x63c85b8, 0xc0828ccd20}, 0xc02dc27f40, 0xc07d93eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0828ccd20}, 0x660089, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023b5a800, {0x63c85b8, 0xc0828ccd20}, 0xc0827a5600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0828ccc60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0828ccc60}, {0xc0032fa8f0, 0x660089}, 0xc0032fa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0828ccc60}, {0xc00102e4e0}, 0xc0827a5500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0828ccc00}, 0xc0827a5500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01f1b6f30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806062 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc024953ce0}, 0xc06749e800, 0xc0765b1f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc024953ce0}, 0xc06749e800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104be44f22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104be44f22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024953c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024953c50}, {0xc00716b3f8, 0x660089}, 0xc00716b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024953c50}, {0x0}, 0xc06749e500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024953bc0}, 0xc06749e500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}, 0xc058a9df80, 0xc08aad8dc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}, 0x3, 0xc0270884c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00716b980}}}, {0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}, 0xc058a9df80, 0xc0270884c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc024953bc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024953bc0}, {0x40, 0xc00716bb70}, 0xc00716bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc024953bc0}, {0x4e9d7a0, 0xc06749e500}, 0xc058a9df80, 0xc08aad8dc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc024953bc0}, 0xc01d942060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0784b70e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0784b70e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28121 [select]: github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowScheduler).Start.func1({0xc00249c600, 0xc02b0d73e0}) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow_scheduler.go:331 +0x25c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28483 [select]: github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal({{0x63c85b8, 0xc01cbda3c0}, 0xc013411f80, 0xc0152fa120}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:694 +0xbf github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:723 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc00167cf00, {0x63c8510, 0xc017d65800}, {{0xc01eff3cd8, 0x1, 0x8}, {0xc01eff3ce0, 0x1, 0x8}}, {0x1742d9fda7f82f8c, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:519 +0xd84 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc00167cf00, {0x63c8510, 0xc017d65800}, 0xc01504e9c0, {{0xc01eff3cd8, 0x1, 0x8}, {0xc01eff3ce0, 0x1, 0x8}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:336 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1({0x63c8510, 0xc017d65800}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:120 +0xbe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219803465 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02efcb000, {0x63c85b8, 0xc06bdf3680}, 0xc056934b00, 0xc077e9e780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02efcb000, {0x63c85b8, 0xc06bdf3680}, 0xc056934b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d3cba74, 0x0, 0x0}, 0xc03b0d7b40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d3cba74, 0x0, 0x0}, 0xc03b0d7b40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06bdf35f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06bdf35f0}, {0xc0764953f8, 0x660089}, 0xc076495400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06bdf35f0}, {0x7fe3c00c4de8}, 0xc0571e9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06bdf3560}, 0xc0571e9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}, 0xc078c2e4e0, 0xc02629d488) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}, 0x3, 0xc064298f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076495980}}}, {0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}, 0xc078c2e4e0, 0xc064298f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06bdf3560}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06bdf3560}, {0x40, 0xc076495b70}, 0xc076495ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06bdf3560}, {0x4e9d7a0, 0xc0571e9f00}, 0xc078c2e4e0, 0xc02629d488) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06bdf3560}, 0xc02b685e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03a3f39e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03a3f39e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 204400907 [sync.Cond.Wait, 19 minutes]: sync.runtime_notifyListWait(0xc06e7d93d0, 0xd) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc038a3e000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc038a3e660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03d4a1900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03d4a1900, {0x63c8510, 0xc06e7d9400}, 0xc08f243dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01ae815e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06e7d9400}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01ae815e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810822 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc072c81920}, 0xc0c884dbf0, 0x1, 0xc00e9fb0a8, 0x519bbc, 0xc092bd2f00, 0xc0730dda40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04d3bb260, {0x63c85b8, 0xc072c81920}, 0xc063c36120, 0xc00e9fb1a8, 0x1, 0x0, 0x0, 0xc092bd2f00, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc072c81920}, 0xc092bd2ea0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc045559d40, 0xa8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc072c81920}, 0x63523b0, 0x412ad318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc072c81920}, {0x0, {0x174301106f56a1d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc072c81920}, 0xc06886fae0, 0xc00e9fb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc072c81920}, 0x660089, {0x0, {0x174301106f56a1d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc072c81920}, 0xc047481800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f56a1d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f56a1d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301106f56a1d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072c81890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072c81890}, {0xc00e9fd3f8, 0x660089}, 0xc00e9fd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072c81890}, {0x7fe331725b30}, 0xc047481600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072c81770}, 0xc047481600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}, 0xc048f64900, 0xc064a91308) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}, 0x3, 0xc0bd512700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e9fd980}}}, {0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}, 0xc048f64900, 0xc0bd512700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc072c81770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072c81770}, {0x40, 0xc00e9fdb70}, 0xc00e9fdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc072c81770}, {0x4e9d7a0, 0xc047481600}, 0xc048f64900, 0xc064a91308) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc072c81770}, 0xc0285486c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc050432360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc050432360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794792 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc05ec81080}, 0xc07e121cb0, {0x646cc48, 0xc05ec810e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc05ec81080}, 0xedb7a834e, 0xc025e77d40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc05ec81080}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03f4e3530}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc07e121ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc03f4e3530}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800047 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0140fe5a0}, 0xc04da04cb0, {0x646cc48, 0xc0140fe600}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0140fe5a0}, 0xedb7a834e, 0xc0ae9a0b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0140fe5a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02712b5c0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc04da04ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc02712b5c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28501 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc0110d9cf8) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01cc2c960) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01cc2c960, {0x63c8510, 0xc017d65800}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00167cf00, {0x63c8510, 0xc017d657c0}, {0xc02c253ce0, 0x1, 0xc01db6f080}, {0xc0110286b8, 0x200c0930, 0xc0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:139 +0x625 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x8, {0x63c8510, 0xc017d657c0}, {0xc02c253ce0, 0x63c8510, 0xc017d653c0}, {0xc02cafa060, 0x1db6f080, 0xc0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:78 +0x45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x63c8510, 0xc017d657c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:285 +0x7a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219812048 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b683cc0, {0x63c85b8, 0xc0725427e0}, 0xc01dc61ef0, 0x1, 0x0, 0x1, 0xc00f88b260, 0xc04da983c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0725427e0}, 0x120, 0xc036576728, 0xf88b200, 0x0, 0x660069, 0xc00f88b260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b683cc0, {0x63c85b8, 0xc0725427e0}, 0xc00f88b200, {{{{0x0, 0x0}, {0xc073d50ea0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0725427e0}, 0x6352410, 0x204d2440) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b5e9ea0, {0x63c85b8, 0xc0725427e0}, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b5e9ea0, {0x63c85b8, 0xc0725427e0}, 0xc06e436000, 0xc07b0d4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0725427e0}, 0x660089, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b6e1800, {0x63c85b8, 0xc0725427e0}, 0xc045013800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072542720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072542720}, {0xc00309e8f0, 0x660089}, 0xc00309e8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072542720}, {0xc0601ffc80}, 0xc045013700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0725426c0}, 0xc045013700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc072542180}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812661 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804877 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc0452c7c20}, 0xc025eb3518, 0x1, 0xc077b490a8, 0x519bbc, 0xc06f6ff920, 0xc058d9b4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0452c7c20}, 0x120, 0xc077b49230, 0x6f6ff8c0, 0xc01967cb60, 0x18, 0xc06f6ff920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc0452c7c20}, 0xc06f6ff8c0, {{{{0x0, 0x0}, {0xc0557bc900, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0452c7c20}, 0x6352410, 0x34331a50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc0452c7c20}, {0xc06f6ff7a0, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc0452c7c20}, 0xc081f74640, 0xc077b499f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0452c7c20}, 0x660089, {0xc06f6ff7a0, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc0452c7c20}, 0xc06b515500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0452c7b30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0452c7b30}, {0xc077b4b3f8, 0x660089}, 0xc077b4b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0452c7b30}, {0x68}, 0xc06b515200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0452c7a40}, 0xc06b515200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}, 0xc065a4a620, 0xc017456618) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}, 0x3, 0xc05c1410c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc077b4b980}}}, {0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}, 0xc065a4a620, 0xc05c1410c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0452c7a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0452c7a40}, {0x40, 0xc077b4bb70}, 0xc077b4bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}, 0x203017, 0x203017) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0452c7a40}, {0x4e9d7a0, 0xc06b515200}, 0xc065a4a620, 0xc017456618) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0452c7a40}, 0xc03ec078c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85c480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85c480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805395 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc078634390}, 0xc0525c1050, 0x1, 0x708017ded0aca9, 0x37e11d600, 0xc05ad4f6e0, 0xc05a99d140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc078634390}, 0x120, 0xc094197230, 0x5ad4f680, 0xc01967cea0, 0x18, 0xc05ad4f6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc078634390}, 0xc05ad4f680, {{{{0x0, 0x0}, {0xc044135b00, 0x14}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc078634390}, 0x6352410, 0x3a07e000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc078634390}, {0xc05ad4f560, {0x1743011041c6d918, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc078634390}, 0xc05ad62aa0, 0xc0941979f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc078634390}, 0x660089, {0xc05ad4f560, {0x1743011041c6d918, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc078634390}, 0xc08b702f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041c6d918, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041c6d918, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078634300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078634300}, {0xc0941993f8, 0x660089}, 0xc094199400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078634300}, {0x4d939c}, 0xc08b702d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078634270}, 0xc08b702d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}, 0xc0a0c540e0, 0xc0525c1020) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}, 0x3, 0xc0455d1700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc094199980}}}, {0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}, 0xc0a0c540e0, 0xc0455d1700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc078634270}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078634270}, {0x40, 0xc094199b70}, 0xc094199ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}, 0x203011, 0x203011) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc078634270}, {0x4e9d7a0, 0xc08b702d00}, 0xc0a0c540e0, 0xc0525c1020) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc078634270}, 0xc02fd388a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ad4f0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ad4f0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812749 [semacquire]: sync.runtime_SemacquireMutex(0xc04321ae38, 0xcd, 0xc00074ab70) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0x118663c, 0xc04321aea0, 0x11844cd) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc04321af98, 0x11a5314, 0x7fde629f5180, 0x7fe1683d1aa0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x4d866e, 0x9185600}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).init(0xc05437e000, 0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1229 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2157 +0xdb github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a528c0, 0xc04321b188, 0xc09279a3e8, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc09279a500, 0x0, 0xc09279a500) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc09279a398, 0xc0734106e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09279a398, {0xc09755e820, 0x2f, 0x50}, {0xc0755ae380, 0x2f, 0x79}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092798cc0, {0xc0755ae380, 0x2, 0xb}, 0x2c, 0xc8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092798cc0, {0xc09755e820, 0x79, 0xc0612d6f00}, {0xc0755ae380, 0xc0612d6f00, 0x2e}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092798a00, {0xc0755ae380, 0x2f, 0x79}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc07858eea8, {{0xc0612d6f00, 0x2e, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06b646200, {{0xc04d61c150, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc06b731c00, {0x63c85b8, 0xc079e52030}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc079e52030}, {0x64a6560, 0xc06b646200}, {0xc04d61c150, 0xc00bc1fa40, 0x30}, {0x174301108da61675, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc079e52030}, {0x6466ed8, 0xc07858ec00}, {0xc04d61c150, 0x27, 0x30}, {0x174301108da61675, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc046ebe768}, {{0x174301108da61675, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301108da61675, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0645eea00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110ab737b75, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d7c00, {0x63c85b8, 0xc079e52030}, 0xc0645eea00, 0xc030c64fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d7c00, {0x63c85b8, 0xc079e52030}, 0xc0645eea00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108da61675, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108da61675, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f2c5f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f2c5f80}, {0xc0432213f8, 0x660089}, 0xc043221400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f2c5f80}, {0x1}, 0xc0645ee700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f2c5d10}, 0xc0645ee700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}, 0xc03ebddba0, 0xc031cfb9f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}, 0x3, 0xc015b9f040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043221980}}}, {0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}, 0xc03ebddba0, 0xc015b9f040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f2c5d10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f2c5d10}, {0x40, 0xc043221b70}, 0xc043221ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f2c5d10}, {0x4e9d7a0, 0xc0645ee700}, 0xc03ebddba0, 0xc031cfb9f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f2c5d10}, 0xc01c342fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc045414ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc045414ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802569 [runnable]: sync.runtime_SemacquireMutex(0xc09750a690, 0x8f, 0x7fd5811b0880) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc09750a538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc09750a560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc03a448900, {0xffffffffffffff15, 0xc09750a6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc03a448900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc03a448900, {0xc0138ec780, 0x32, 0x77}, {0xc078fee8c0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0942f1e98, {0xc0138ec780, 0x32, 0x77}, {0xc078fee8c0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0942f07c0, {0xc078fee8c0, 0x32, 0x77}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0942f07c0, {0xc0138ec780, 0x32, 0xc078fee8c0}, {0xc078fee8c0, 0xc043bad140, 0x31}, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0942f0500, {0xc078fee8c0, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0484bb9a8, {{0xc043bad140, 0x31, 0x40}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0503de500, {{0xc060158db0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc051fcf000, {0x63c85b8, 0xc057474de0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc057474de0}, {0x64a6560, 0xc0503de500}, {0xc060158db0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc057474de0}, {0x6466ed8, 0xc0484bb700}, {0xc060158db0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0400ab3e0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc041b61700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f1e400, {0x63c85b8, 0xc057474de0}, 0xc041b61700, 0xc054757b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f1e400, {0x63c85b8, 0xc057474de0}, 0xc041b61700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057474d50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057474d50}, {0xc0936988f0, 0x660089}, 0xc0936988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057474d50}, {0xc00102e4e0}, 0xc041b61600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057474cf0}, 0xc041b61600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220930}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28320 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0118d0960, {0xc0002e73f0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0118d0960, {0xc0002e73f0, 0xc0195781c8, 0xc01f7f5a80}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc01cbb4f30, {0xc0002e73f0, 0xc01f7f5af8, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc01cbb4f30}, {0xc0002e73f0, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00b38ad80, {0xc0002e73f0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc0002e73e0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x58, 0xc00b38ad80, {0x0, 0x0}, 0x7fffffff, 0xc01f7f5da8, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0xc06123bda0, {0x0, 0x0}, {0x4cfd920, 0xc06123bda0}, 0x203018, 0xc01f7f5da8, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc0009fcea0, {0x4cfd920, 0xc06123bda0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb.(*sideTransportPushUpdatesServer).Recv(0xc0147980b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:365 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run.func1({0x63c85b8, 0xc01cbb5350}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:304 +0xc2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219797833 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035040, {0x63c85b8, 0xc0161f55f0}, 0xc0bba62960, 0x1, 0xc0918a78a8, 0x519bbc, 0xc01cf3f020, 0xc0d19a9020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01e5e19e0, {0x63c85b8, 0xc0161f55f0}, 0xc01e5e1a28, 0xc0918a79a8, 0x41, 0x60, 0xffffffffffffffff, 0xc01cf3f020, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035040, {0x63c85b8, 0xc0161f55f0}, 0xc01cf3efc0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc077e2f680, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc0161f55f0}, 0x63523b0, 0x3824c7e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd3b0, {0x63c85b8, 0xc0161f55f0}, {0x0, {0x1743011077f568eb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd3b0, {0x63c85b8, 0xc0161f55f0}, 0xc065099cc0, 0xc0918a81f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0161f55f0}, 0x660089, {0x0, {0x1743011077f568eb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc0161f55f0}, 0xc0984dd300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0161f53e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0161f53e0}, {0xc0918a9bf8, 0x660089}, 0xc0918a9c00) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0161f53e0}, {0xc044100930}, 0xc0984dd100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0161f52f0}, 0xc0984dd100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x1743011077f568eb, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc044100a80}, 0x0, 0xc058bb5700) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc044100a80}, 0x19) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0078dea50, {0x63c85b8, 0xc044100a80}, 0xc0918ac638, {{0x1743011076c2b05c, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:392 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0x0, {0x63c85b8, 0xc044100a80}, 0xc04474e398, {{0x1743011076c2b05c, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x165 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc02dfdd400, {0x63c85b8, 0xc044100a80}, {0xc01d33f680, {0x1743011058f54b5c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:499 +0x533 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc044100a80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc02dfdd400, {0x63c85b8, 0xc044100a80}, {0xc01d33f680, {0x1743011058f54b5c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd3b0, {0x63c85b8, 0xc044100a80}, 0xc02c9fe0a0, 0xc0918ad4e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044100a80}, 0xc02c9fe0a0, {0xc01d33f680, {0x1743011058f54b5c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc044100a80}, 0xc099f8bb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058f54b5c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058f54b5c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0441009f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0441009f0}, {0xc0918af380, 0x660089}, 0xc0918af388) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0441009f0}, {0xc00102e4e0}, 0xc099f8ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044100990}, 0xc099f8ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc02a22f4a0, {0x63c85b8, 0xc071254540}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc02a22f4a0, {0x63c85b8, 0xc071254540}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc000497770, {0x63c85b8, 0xc071254540}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc00f675500, {0x63c85b8, 0xc071254540}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc089cb06e0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:264 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc06386f7c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc03ead1320) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0aea93800) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc0b285c580, {0x63c8510, 0xc06386f700}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc04c552730, {0x63c8510, 0xc06386f700}, 0xc03a95d290) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc06386f540}, 0xc07b7ed880, 0xc03a95d290, 0xc08d132100, 0xc04dc09180, 0xc089e18fe8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc06386f540}, 0xc089e18cf0, 0xc07b7ed880, 0xc06d19e6c0, {{0x63ca960, 0xc03281a8c0}, 0x0}, 0xc04dc09180) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc089e18600, {0x63c8510, 0xc06386f540}, 0xc089e18cf0, 0xc06386f540, {0x7fe42b790cf0, 0xc06d19e6c0}, 0x0, 0xc03ead11f8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc089e18600, {0x63c8510, 0xc06386f540}, 0xc089e18cf0, {0x7fe42b790cf0, 0xc06d19e6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc089e18600, {0x63c85b8, 0xc02d17cb10}, {{0x63f8d78, 0xc00e90b310}, {0xc000f29010, 0x174}, 0x1, 0x1}, 0xc0729b70e0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc089e18600, {0x63c85b8, 0xc02d17cb10}, {{0x63f8d78, 0xc00e90b310}, {0xc000f29010, 0x174}, 0x1, 0x1}, 0xc0729b70e0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc089e18600, {0x63c85b8, 0xc02d17cb10}, {0xc0729b70e0, {0xc0760f2f40, 0x1, 0x1}, {0xc064aaa840, 0x12, 0x12}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc000f29b00, 0x0}, 0x0, {0x21df005, 0xedb7a8350, 0x0}, 0x1}, 0xc089e18600, 0xc0918b5980, 0xc0918b5970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc089e18600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc089e18600, {0x63c8510, 0xc070aee600}, 0xc095ad7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc044b7ba30) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc070aee600}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc044b7ba30) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802264 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc095158548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0951585b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc092bd05f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc05a70d180, {0xc03115b1d0, 0x2b, 0x2b}, {0xc03115b1a0, 0x2b, 0x2b}, 0x1, 0x5) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092bd0488, {0xc03115b1d0, 0x1, 0x2b}, {0xc03115b1a0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092bd0488, {0xc03115b1d0, 0x2b, 0x2b}, {0xc03115b1a0, 0x2b, 0x2b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092bcfbc0, {0xc03115b1a0, 0x2b, 0x2b}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092bcfbc0, {0xc03115b1d0, 0x2b, 0xc03115b1a0}, {0xc03115b1a0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092bcf900, {0xc03115b1a0, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06c050588, {{0xc0630d0ff0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc039e61400, {{0xc0630d0ff0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f584c00, {0x63c85b8, 0xc04e4bdec0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04e4bdec0}, {0x64a6560, 0xc039e61400}, {0xc0630d0ff0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04e4bdec0}, {0x6466ed8, 0xc06c050580}, {0xc0630d0ff0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0a6a834d0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01247f600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020edf000, {0x63c85b8, 0xc04e4bdec0}, 0xc01247f600, 0xc0c91200a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020edf000, {0x63c85b8, 0xc04e4bdec0}, 0xc01247f600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e4bde30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e4bde30}, {0xc0025128f0, 0x660089}, 0xc0025128f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e4bde30}, {0xc00102e4e0}, 0xc01247f500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e4bddd0}, 0xc01247f500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0bb6395f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811865 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc0654c14a0}, 0xc03f861ae8, 0x1, 0xc0117330a8, 0x519bbc, 0xc039070d80, 0xc05a760420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0654c14a0}, 0x1e0, 0xc011733230, 0x39070d20, 0x0, 0x660069, 0xc039070d80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc0654c14a0}, 0xc039070d20, {{{{0x0, 0x0}, {0xc06c58a480, 0x3f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x631bca0, {0x63c85b8, 0xc0654c14a0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc0654c14a0}, {0xc0469c4240, {0x1743011065ea6c77, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc0654c14a0}, 0xc0d04523c0, 0xc0117339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0654c14a0}, 0x660089, {0xc0469c4240, {0x1743011065ea6c77, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0654c14a0}, 0xc06e40ea00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ea6c77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ea6c77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0654c1410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0654c1410}, {0xc0117353f8, 0x660089}, 0xc011735400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0654c1410}, {0x7fe3ba6bf2b0}, 0xc06e40e700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0654c1380}, 0xc06e40e700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}, 0xc0428fd640, 0xc03f861ed8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}, 0x3, 0xc0450f8a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc011735980}}}, {0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}, 0xc0428fd640, 0xc0450f8a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0654c1380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0654c1380}, {0x40, 0xc011735b70}, 0xc011735ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0654c1380}, {0x4e9d7a0, 0xc06e40e700}, 0xc0428fd640, 0xc03f861ed8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0654c1380}, 0xc0bac65680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0469c4000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0469c4000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806191 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc08992c030}, 0xc08aad9770, 0x1, 0x0, 0x1, 0xc0468b2de0, 0xc0468b2ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08992c030}, 0x120, 0xc00a7a8728, 0x468b2d80, 0x0, 0x660069, 0xc0468b2de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc08992c030}, 0xc0468b2d80, {{{{0x0, 0x0}, {0xc0524f4000, 0x25}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08992c030}, 0x6352410, 0x24a009a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc08992c030}, {0xc07398c240, {0x174301104cf73160, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc08992c030}, 0xc07bbec1e0, 0xc005be6ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08992c030}, 0x660089, {0xc07398c240, {0x174301104cf73160, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc08992c030}, 0xc01065e900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104cf73160, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104cf73160, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06859ff80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06859ff80}, {0xc0d68348f0, 0x660089}, 0xc0d68348f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06859ff80}, {0xc00102e4e0}, 0xc01065e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06859ff20}, 0xc01065e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc053b038c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809227 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024c49b80, {0x63c85b8, 0xc053cc1920}, 0xc0234ebc80, 0x1, 0xc0136b30a8, 0x519bbc, 0xc034ef44e0, 0xc047a235c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc053cc1920}, 0x120, 0xc0136b3230, 0x34ef4480, 0x0, 0x660069, 0xc034ef44e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024c49b80, {0x63c85b8, 0xc053cc1920}, 0xc034ef4480, {{{{0x0, 0x0}, {0xc07ca00000, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc053cc1920}, 0x6352410, 0x47996708) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024c054f0, {0x63c85b8, 0xc053cc1920}, {0xc034ef4360, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024c054f0, {0x63c85b8, 0xc053cc1920}, 0xc04c398140, 0xc0136b39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc053cc1920}, 0x660089, {0xc034ef4360, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024cab000, {0x63c85b8, 0xc053cc1920}, 0xc058b81400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053cc1800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053cc1800}, {0xc0136b53f8, 0x660089}, 0xc0136b5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053cc1800}, {0xc0015221e0}, 0xc058b81100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053cc1770}, 0xc058b81100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}, 0xc06acbb040, 0xc0418102b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}, 0x3, 0xc01fbf91c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0136b5980}}}, {0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}, 0xc06acbb040, 0xc01fbf91c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc053cc1770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053cc1770}, {0x40, 0xc0136b5b70}, 0xc0136b5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc053cc1770}, {0x4e9d7a0, 0xc058b81100}, 0xc06acbb040, 0xc0418102b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc053cc1770}, 0xc01c7833e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330a7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330a7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812028 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc046232960}, 0xc0cc964b28, 0x1, 0xc00cc550a8, 0x519bbc, 0xc00e5e0600, 0xc04c3e9020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046232960}, 0x120, 0xc00cc55230, 0xe5e05a0, 0x0, 0x660069, 0xc00e5e0600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc046232960}, 0xc00e5e05a0, {{{{0x0, 0x0}, {0xc02875a900, 0x35}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046232960}, 0x6352410, 0x204d2670) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc046232960}, {0xc00e5e0480, {0x174301107847188e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc046232960}, 0xc031a60dc0, 0xc00cc559f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046232960}, 0x660089, {0xc00e5e0480, {0x174301107847188e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc046232960}, 0xc067750200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107847188e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107847188e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0462328d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0462328d0}, {0xc00cc573f8, 0x660089}, 0xc00cc57400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0462328d0}, {0x0}, 0xc067750000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046232840}, 0xc067750000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}, 0xc056442100, 0xc03c1ea918) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}, 0x3, 0xc032964200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00cc57980}}}, {0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}, 0xc056442100, 0xc032964200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046232840}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046232840}, {0x40, 0xc00cc57b70}, 0xc00cc57ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046232840}, {0x4e9d7a0, 0xc067750000}, 0xc056442100, 0xc03c1ea918) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046232840}, 0xc0314bb7a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc00e5e0240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc00e5e0240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812725 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc04f5ed8c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc07b81b500, 0xc04007d168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04f5ed8c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc07b81b500, {0xc004156580, 0x29, 0x74}, {0xc01698ccb0, 0x29, 0x6d}, 0x0, 0xd1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0bd73d038, {0xc004156580, 0x1, 0x6d}, {0xc01698ccb0, 0xc0bd73c000, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0bd73d038, {0xc004156580, 0x29, 0x74}, {0xc01698ccb0, 0x29, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0bd73c2c0, {0xc01698ccb0, 0x0, 0xc04007d4e0}, 0x121ea30, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0bd73c2c0, {0xc004156580, 0x29, 0x6d}, {0xc01698ccb0, 0x28, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0bd73c000, {0xc01698ccb0, 0x29, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc077169608, {{0xc051de1980, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0517d3a00, {{0xc051de1980, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0777f6400, {0x63c85b8, 0xc062cde630}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062cde630}, {0x64a6560, 0xc0517d3a00}, {0xc051de1980, 0x0, 0x30}, {0x1743011093a0ad78, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062cde630}, {0x6466ed8, 0xc077169600}, {0xc051de1980, 0x28, 0x30}, {0x1743011093a0ad78, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04459e870}, {{0x1743011093a0ad78, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011093a0ad78, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc050ef2600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b16e1278, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0312b0400, {0x63c85b8, 0xc062cde630}, 0xc050ef2600, 0xc0895b4dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0312b0400, {0x63c85b8, 0xc062cde630}, 0xc050ef2600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011093a0ad78, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011093a0ad78, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cde570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cde570}, {0xc0400833f8, 0x660089}, 0xc040083400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cde570}, {0x7fdd4a36fa20}, 0xc050ef2300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cde4e0}, 0xc050ef2300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}, 0xc06ac12a80, 0xc041139f80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}, 0x3, 0xc01c0bd9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc040083980}}}, {0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}, 0xc06ac12a80, 0xc01c0bd9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cde4e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cde4e0}, {0x40, 0xc040083b70}, 0xc040083ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cde4e0}, {0x4e9d7a0, 0xc050ef2300}, 0xc06ac12a80, 0xc041139f80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cde4e0}, 0xc049257440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a57a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a57a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807619 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc02cf77a40}, 0xc05fed7d88, 0x1, 0xc00646f0a8, 0x519bbc, 0xc08718e060, 0xc084e38060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02cf77a40}, 0x1e0, 0xc00646f230, 0x8718e000, 0x0, 0x660069, 0xc08718e060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc02cf77a40}, 0xc08718e000, {{{{0x0, 0x0}, {0xc03cc18d80, 0x34}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc02cf77a40}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc02cf77a40}, {0xc08f42f680, {0x174301105781e9b4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc02cf77a40}, 0xc031691ae0, 0xc00646f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02cf77a40}, 0x660089, {0xc08f42f680, {0x174301105781e9b4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc02cf77a40}, 0xc09903a900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105781e9b4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105781e9b4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02cf779b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02cf779b0}, {0xc0064713f8, 0x660089}, 0xc006471400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02cf779b0}, {0x4e4148435f415441}, 0xc09903a700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02cf77920}, 0xc09903a700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}, 0xc01e64a920, 0xc0603c2510) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}, 0x3, 0xc0619ff380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006471980}}}, {0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}, 0xc01e64a920, 0xc0619ff380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02cf77920}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02cf77920}, {0x40, 0xc006471b70}, 0xc006471ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02cf77920}, {0x4e9d7a0, 0xc09903a700}, 0xc01e64a920, 0xc0603c2510) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02cf77920}, 0xc02ed0b560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0579f4000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0579f4000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807680 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0701c1e90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810309 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024c58400, {0x63c85b8, 0xc073a6edb0}, 0xc082397400, 0xc05b113900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024c58400, {0x63c85b8, 0xc073a6edb0}, 0xc082397400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cc47eaf, 0x0, 0x0}, 0xc032ab3ec0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cc47eaf, 0x0, 0x0}, 0xc032ab3ec0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073a6ed20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073a6ed20}, {0xc0272a53f8, 0x660089}, 0xc0272a5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073a6ed20}, {0x7fe3b8b316d0}, 0xc082397200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073a6ec60}, 0xc082397200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}, 0xc078800ee0, 0xc089245188) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}, 0x3, 0xc024365c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0272a5980}}}, {0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}, 0xc078800ee0, 0xc024365c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073a6ec60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073a6ec60}, {0x40, 0xc0272a5b70}, 0xc0272a5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073a6ec60}, {0x4e9d7a0, 0xc082397200}, 0xc078800ee0, 0xc089245188) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073a6ec60}, 0xc0239d3c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0610be360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0610be360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802967 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01dd05000, {0x63c85b8, 0xc04aee8420}, 0xc042123b00, 0xc023a39d60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc04aee8420}, 0xc042123b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011028e5e8ca, 0x0, 0x0}, 0xc03d5d4660, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011028e5e8ca, 0x0, 0x0}, 0xc03d5d4660, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04aee8390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aee8390}, {0xc06dccb3f8, 0x660089}, 0xc06dccb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04aee8390}, {0x679d1b8e091a0192}, 0xc042123900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04aee8300}, 0xc042123900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}, 0xc024d863a0, 0xc04025d1a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}, 0x3, 0xc06e42b040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06dccb980}}}, {0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}, 0xc024d863a0, 0xc06e42b040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04aee8300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aee8300}, {0x40, 0xc06dccbb70}, 0xc06dccbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04aee8300}, {0x4e9d7a0, 0xc042123900}, 0xc024d863a0, 0xc04025d1a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04aee8300}, 0xc045f6b4a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05fe76240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05fe76240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800139 [runnable]: sync.runtime_SemacquireMutex(0xc095890690, 0x8f, 0x7fe29cc3abc0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc095890538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc095890560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc06d5dfb00, {0xffffffffffffff15, 0xc0958906c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc06d5dfb00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc06d5dfb00, {0xc0226ea0a0, 0x32, 0x48}, {0xc05ff92f00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091b68898, {0xc0226ea0a0, 0x32, 0x48}, {0xc05ff92f00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091b671c0, {0xc05ff92f00, 0x74, 0x48}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091b671c0, {0xc0226ea0a0, 0x74, 0xc05ff92f00}, {0xc05ff92f00, 0xc057aeebe0, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091b66f00, {0xc05ff92f00, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc039026928, {{0xc057aeebe0, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc040a2b500, {{0xc059b194d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04d651400, {0x63c85b8, 0xc0254af620}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0254af620}, {0x64a6560, 0xc040a2b500}, {0xc059b194d0, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0254af620}, {0x6466ed8, 0xc039026680}, {0xc059b194d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc065d81038}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05f9f1900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a33c00, {0x63c85b8, 0xc0254af620}, 0xc05f9f1900, 0xc05b1130e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a33c00, {0x63c85b8, 0xc0254af620}, 0xc05f9f1900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0254af590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0254af590}, {0xc0031068f0, 0x660089}, 0xc0031068f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0254af590}, {0xc00102e4e0}, 0xc05f9f1800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0254af4a0}, 0xc05f9f1800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d470}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219150618 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc019b855e0, {0xc02acf3978, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc019b855e0, {0xc02acf3978, 0xc027ff96f8, 0xc021977a68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc03b84c450, {0xc02acf3978, 0xc021977ae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc03b84c450}, {0xc02acf3978, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0597b4ea0, {0xc02acf3978, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc02acf3968, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x58, 0xc0597b4ea0, {0x0, 0x0}, 0x7fffffff, 0xc021977d90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x63474f8, {0x7fe42ab594c0, 0xc000808f40}, 0x1, {0x0, 0x0}, {0x4cfe5a0, 0xc017dae600}, 0xc0082db200, 0xc021977d90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc04ee3ef70, {0x4cfe5a0, 0xc017dae600}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc014670110) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc014670110}, 0xc033d160b0, {0x63c85b8, 0xc050e35080}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc050e35080}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219798975 [runnable]: sync.runtime_SemacquireMutex(0xc0934f0690, 0x8f, 0x7fe1e4194080) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0934f0538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0934f0560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc00c6a7200, {0xffffffffffffff15, 0xc0934f06c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc00c6a7200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc00c6a7200, {0xc02afda900, 0x32, 0x79}, {0xc0a3188fc0, 0x32, 0x52}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09395de98, {0xc02afda900, 0x32, 0x79}, {0xc0a3188fc0, 0x32, 0x52}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09395c7c0, {0xc0a3188fc0, 0x52, 0x79}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09395c7c0, {0xc02afda900, 0x52, 0xc0a3188fc0}, {0xc0a3188fc0, 0xc06b6f7880, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09395c500, {0xc0a3188fc0, 0x32, 0x52}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc05ba57e28, {{0xc06b6f7880, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc064f96300, {{0xc04cd1b890, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc035bdd000, {0x63c85b8, 0xc0738df650}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0738df650}, {0x64a6560, 0xc064f96300}, {0xc04cd1b890, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0738df650}, {0x6466ed8, 0xc05ba57b80}, {0xc04cd1b890, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc03c5641e0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0563c9d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02072b800, {0x63c85b8, 0xc0738df650}, 0xc0563c9d00, 0xc055b22fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02072b800, {0x63c85b8, 0xc0738df650}, 0xc0563c9d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0738df5c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0738df5c0}, {0xc0031068f0, 0x660089}, 0xc0031068f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0738df5c0}, {0xc00102e4e0}, 0xc0563c9c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0738df560}, 0xc0563c9c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06e0bb710}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 4696 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc0153e3ac0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc0124d5d20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}, 0xc016613380, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}, 0x3, 0xc0234d4440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0xc007d818c0}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}, 0xc016613380, 0xc0234d4440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d4cab10}, {0x460ef4, 0xc002ca2bc8}, 0xc002ca2b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}, 0xc016613380, 0xc0234d4440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc0016e32b0}, 0xc016613380, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc019572680}, 0xc01300f440, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019572680}, 0xc01300f440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810150 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e115900, {0x63c85b8, 0xc0199c7560}, 0xc0738dc540, 0x1, 0xc0146370a8, 0x519bbc, 0xc08979df00, 0xc05fad71a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc072f777a0, {0x63c85b8, 0xc0199c7560}, 0xc08979df00, 0xc0146371a8, 0x64, 0xa0, 0x0, 0xc08979df00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e115900, {0x63c85b8, 0xc0199c7560}, 0xc08979dc00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc007f44c60, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0199c7560}, 0x63521a0, 0x6b245688) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dd81a40, {0x63c85b8, 0xc0199c7560}, {0xc06d26e480, {0x1743011008d85bd8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dd81a40, {0x63c85b8, 0xc0199c7560}, 0xc01c696960, 0xc0146379f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0199c7560}, 0x660089, {0xc06d26e480, {0x1743011008d85bd8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e1c2400, {0x63c85b8, 0xc0199c7560}, 0xc053812600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011008d85bd8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011008d85bd8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0199c74d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0199c74d0}, {0xc0146393f8, 0x660089}, 0xc014639400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0199c74d0}, {0xc065482230}, 0xc053812300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0199c7410}, 0xc053812300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}, 0xc0701d4f60, 0xc0bb3a9c20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}, 0x3, 0xc02853a640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc014639980}}}, {0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}, 0xc0701d4f60, 0xc02853a640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0199c7410}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0199c7410}, {0x40, 0xc014639b70}, 0xc014639ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0199c7410}, {0x4e9d7a0, 0xc053812300}, 0xc0701d4f60, 0xc0bb3a9c20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0199c7410}, 0xc0218d7560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a72a7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a72a7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 16180 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc0020abcf8) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc0231819e0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0231819e0, {0x63c8510, 0xc002936240}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00167cf00, {0x63c8510, 0xc0036c5c00}, {0xc018b70090, 0x1, 0xc02b6e2400}, {0x0, 0xb970e0, 0x0}, 0x1, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:139 +0x625 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x63491b8, {0x63c8510, 0xc0036c5c00}, {0xc018b70090, 0x63c85b8, 0xc0231ebec0}, {0xf0f77dcdb6d8f00, 0x63c85b8, 0x0}, 0x1, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:78 +0x45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x63c8510, 0xc0036c5c00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:285 +0x7a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219806935 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ba0e500, {0x63c85b8, 0xc092c06ae0}, 0xc04e68b698, 0x1, 0xc004a790a8, 0x519bbc, 0xc00b2935c0, 0xc097e05260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc092c06ae0}, 0x120, 0xc004a79230, 0xb293560, 0x0, 0x660069, 0xc00b2935c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ba0e500, {0x63c85b8, 0xc092c06ae0}, 0xc00b293560, {{{{0x0, 0x0}, {0xc00635afc0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc092c06ae0}, 0x6352410, 0x1b928ab8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b9f85f0, {0x63c85b8, 0xc092c06ae0}, {0xc00b293440, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b9f85f0, {0x63c85b8, 0xc092c06ae0}, 0xc0ce753860, 0xc004a799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc092c06ae0}, 0x660089, {0xc00b293440, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ba1a000, {0x63c85b8, 0xc092c06ae0}, 0xc07a042f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc092c06a50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092c06a50}, {0xc004a7b3f8, 0x660089}, 0xc004a7b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc092c06a50}, {0xa1fec4a5abc5080a}, 0xc07a042d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc092c069c0}, 0xc07a042d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}, 0xc0907f9a40, 0xc04e68bd28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}, 0x3, 0xc090f525c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004a7b980}}}, {0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}, 0xc0907f9a40, 0xc090f525c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc092c069c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092c069c0}, {0x40, 0xc004a7bb70}, 0xc004a7bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc092c069c0}, {0x4e9d7a0, 0xc07a042d00}, 0xc0907f9a40, 0xc04e68bd28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc092c069c0}, 0xc04d3bcc60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590b85a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590b85a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810232 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ade7c00, {0x63c85b8, 0xc00f643f20}, 0xc0450e8300, 0xc07beecfa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ade7c00, {0x63c85b8, 0xc00f643f20}, 0xc0450e8300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cced37c, 0x0, 0x0}, 0xc022a471a0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cced37c, 0x0, 0x0}, 0xc022a471a0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc00f643e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00f643e90}, {0xc00b45f3f8, 0x660089}, 0xc00b45f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc00f643e90}, {0x7fe13dd08ef0}, 0xc0450e8100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc00f643d70}, 0xc0450e8100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}, 0xc01eb5f360, 0xc0298190f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}, 0x3, 0xc05914d540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b45f980}}}, {0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}, 0xc01eb5f360, 0xc05914d540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc00f643d70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00f643d70}, {0x40, 0xc00b45fb70}, 0xc00b45fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc00f643d70}, {0x4e9d7a0, 0xc0450e8100}, 0xc01eb5f360, 0xc0298190f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc00f643d70}, 0xc02ec57560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0628686c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0628686c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812466 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc00769bd50, {0x63c85b8, 0xc072bf00c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc072bf00c0}, {0x4f67ad2, 0x10beec0}, 0xc019549fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219806548 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc08aa08240}, 0xc0745a0810, 0x1, 0xc00c8350a8, 0x519bbc, 0xc04028aba0, 0xc055da7920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08aa08240}, 0x120, 0xc00c835230, 0x4028ab40, 0x0, 0x660069, 0xc04028aba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc08aa08240}, 0xc04028ab40, {{{{0x0, 0x0}, {0xc09ecf18c0, 0x106}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08aa08240}, 0x6352410, 0xf066e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc08aa08240}, {0xc04028aa20, {0x174301104e8e23c0, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc08aa08240}, 0xc061527a40, 0xc00c8359f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08aa08240}, 0x660089, {0xc04028aa20, {0x174301104e8e23c0, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc08aa08240}, 0xc062701100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e8e23c0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e8e23c0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08aa081b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08aa081b0}, {0xc00c8373f8, 0x660089}, 0xc00c837400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08aa081b0}, {0x0}, 0xc062700f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08aa08120}, 0xc062700f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}, 0xc08dae37c0, 0xc0745a16f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}, 0x3, 0xc084707dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00c837980}}}, {0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}, 0xc08dae37c0, 0xc084707dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08aa08120}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08aa08120}, {0x40, 0xc00c837b70}, 0xc00c837ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08aa08120}, {0x4e9d7a0, 0xc062700f00}, 0xc08dae37c0, 0xc0745a16f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08aa08120}, 0xc065b99920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04028a900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04028a900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219027482 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a19e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc036e32d80, 0xc00eac5000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc036e32d80, {0xc00eac5000, 0x2b8d, 0x2b8d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc036e32d80, {0xc00eac5000, 0x2b8d, 0xc03238fb80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0412ad8c8, {0xc00eac5000, 0x7eda79, 0xc00eac5005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01f7e1860, {0xc00eac5000, 0x2, 0x2b8d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a90438, {0xc00eac5000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc07b065af8, {0x631b900, 0xc043a90438}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc07b065880, {0x7fe42ab59310, 0xc01f7e1860}, 0x2b8d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc07b065880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc07b065880, {0xc0464a0000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc052e49e18, {0xc0464a0000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0c2114b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0c2114b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0c21150e0, {0x634aac0, 0xc0c2114b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0c2114a80, 0xc051f16c80, 0xc08378942b, 0xc08378942d, {0x635bec0, 0xc02970d200}, 0xc000d6ae00, 0xc03c03cdd4, 0xc083789429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0c2114a80, {0x63c85b8, 0xc07bf26f60}, 0xc051f16c70, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0b1c9e009, 0x8}}, 0x0, 0xc07bf26f00, 0xc07bf26f30, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc07bf26f60}, {0x643d810, 0xc01f7e1860}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02650cc00}, {0x643d810, 0xc01f7e1860}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02650cc00}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801015 [runnable]: sync.runtime_SemacquireMutex(0xc078a18690, 0x8f, 0x7fd597f86580) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc078a18538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc078a18560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc05b12ad80, {0xffffffffffffff15, 0xc078a186c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc05b12ad80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc05b12ad80, {0xc007739f40, 0x32, 0xa0}, {0xc02bd35380, 0x32, 0x7f}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc079d7b298, {0xc007739f40, 0x32, 0xa0}, {0xc02bd35380, 0x32, 0x7f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc079d79bc0, {0xc02bd35380, 0x7f, 0xa0}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc079d79bc0, {0xc007739f40, 0x7f, 0xc02bd35380}, {0xc02bd35380, 0xc01e22c960, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc079d79900, {0xc02bd35380, 0x32, 0x7f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06fb32928, {{0xc01e22c960, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0c6d59c00, {{0xc066d16b10, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc049e20800, {0x63c85b8, 0xc090ec1e00}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc090ec1e00}, {0x64a6560, 0xc0c6d59c00}, {0xc066d16b10, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc090ec1e00}, {0x6466ed8, 0xc06fb32680}, {0xc066d16b10, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07216ce58}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01e34e400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c20000, {0x63c85b8, 0xc090ec1e00}, 0xc01e34e400, 0xc0c2a719a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c20000, {0x63c85b8, 0xc090ec1e00}, 0xc01e34e400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc090ec1d70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090ec1d70}, {0xc0ad8388f0, 0x660089}, 0xc0ad8388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc090ec1d70}, {0xc00102e4e0}, 0xc01e34e300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc090ec1d10}, 0xc01e34e300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a4cc0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28032 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc007bfcaa0, {0xc022c5e928, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc007bfcaa0, {0xc022c5e928, 0xc016f29398, 0xc01fac8a80}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc02c5b2540, {0xc022c5e928, 0xc01fac8af8, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc02c5b2540}, {0xc022c5e928, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc017ee1560, {0xc022c5e928, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc022c5e918, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc01fac8c40, 0xc017ee1560, {0x0, 0x0}, 0x7fffffff, 0xc01fac8da8, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0xc04783f860, {0x0, 0x0}, {0x4cfd920, 0xc04783f860}, 0x203011, 0xc01fac8da8, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc006dfcd00, {0x4cfd920, 0xc04783f860}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb.(*sideTransportPushUpdatesServer).Recv(0xc01c38e570) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:365 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run.func1({0x63c85b8, 0xc02c5b2750}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:304 +0xc2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28229 [select]: github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10({0x63c85b8, 0xc02ca0c930}) github.com/cockroachdb/cockroach/pkg/jobs/registry.go:862 +0x1d0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806184 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ff75000, {0x63c85b8, 0xc03c07c360}, 0xc06749f100, 0xc03289a820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75000, {0x63c85b8, 0xc03c07c360}, 0xc06749f100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c843591, 0x0, 0x0}, 0xc075619ac0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c843591, 0x0, 0x0}, 0xc075619ac0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c07c2d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c07c2d0}, {0xc0076913f8, 0x660089}, 0xc007691400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c07c2d0}, {0x0}, 0xc06749ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c07c240}, 0xc06749ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}, 0xc076b10120, 0xc08aad8fd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}, 0x3, 0xc0270887c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007691980}}}, {0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}, 0xc076b10120, 0xc0270887c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c07c240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c07c240}, {0x40, 0xc007691b70}, 0xc007691ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c07c240}, {0x4e9d7a0, 0xc06749ef00}, 0xc076b10120, 0xc08aad8fd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c07c240}, 0xc01d9426c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03a66eea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03a66eea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219790377 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc01504fec0}, 0xc03463acb0, {0x646cc48, 0xc0bf08a000}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc01504fec0}, 0xedb7a834e, 0xc05ae91200) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc01504fec0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07a44c4b0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc03463aef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc07a44c4b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28211 [select]: github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop(0xc007f32de0, {0x63c8510, 0xc0071fc700}) github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:244 +0x195 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812769 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809258 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021c5a280, {0x63c85b8, 0xc02f410ff0}, 0xc04f9e8948, 0x1, 0xc02d13d0a8, 0x519bbc, 0xc08ac374a0, 0xc07051bec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02f410ff0}, 0x120, 0xc02d13d230, 0x8ac37440, 0x0, 0x660069, 0xc08ac374a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021c5a280, {0x63c85b8, 0xc02f410ff0}, 0xc08ac37440, {{{{0x0, 0x0}, {0xc00f103320, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02f410ff0}, 0x6352410, 0x603b92d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c42460, {0x63c85b8, 0xc02f410ff0}, {0xc08ac37320, {0x1743011063b6cbdc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c42460, {0x63c85b8, 0xc02f410ff0}, 0xc0c9120a00, 0xc02d13d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02f410ff0}, 0x660089, {0xc08ac37320, {0x1743011063b6cbdc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c68c00, {0x63c85b8, 0xc02f410ff0}, 0xc06877ef00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063b6cbdc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063b6cbdc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f410e70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f410e70}, {0xc02d13f3f8, 0x660089}, 0xc02d13f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f410e70}, {0x0}, 0xc06877ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f410de0}, 0xc06877ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}, 0xc07c1847c0, 0xc0430a4840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}, 0x3, 0xc06157aa40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02d13f980}}}, {0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}, 0xc07c1847c0, 0xc06157aa40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f410de0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f410de0}, {0x40, 0xc02d13fb70}, 0xc02d13fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f410de0}, {0x4e9d7a0, 0xc06877ed00}, 0xc07c1847c0, 0xc0430a4840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f410de0}, 0xc037bda660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08ac36d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08ac36d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812216 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035a40, {0x63c85b8, 0xc022c18930}, 0xc0c05c22b8, 0x1, 0xc032dc70a8, 0x519bbc, 0xc00b133080, 0xc05f4408a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06003f320, {0x63c85b8, 0xc022c18930}, 0xc00b133080, 0xc032dc71a8, 0x41, 0x60, 0x4, 0xc00b133080, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035a40, {0x63c85b8, 0xc022c18930}, 0xc00b132fc0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc06fba18c0, 0x8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc022c18930}, 0x6352260, 0x47c52428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd8b0, {0x63c85b8, 0xc022c18930}, {0xc00b132d80, {0x174301104555bd36, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd8b0, {0x63c85b8, 0xc022c18930}, 0xc0adacf360, 0xc032dc79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022c18930}, 0x660089, {0xc00b132d80, {0x174301104555bd36, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e096000, {0x63c85b8, 0xc022c18930}, 0xc08256a500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104555bd36, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104555bd36, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022c188a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022c188a0}, {0xc032dc93f8, 0x660089}, 0xc032dc9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022c188a0}, {0xc007b2d110}, 0xc08256a300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022c18810}, 0xc08256a300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}, 0xc01e517ac0, 0xc0c05c30f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}, 0x3, 0xc03d2c9540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032dc9980}}}, {0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}, 0xc01e517ac0, 0xc03d2c9540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022c18810}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022c18810}, {0x40, 0xc032dc9b70}, 0xc032dc9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022c18810}, {0x4e9d7a0, 0xc08256a300}, 0xc01e517ac0, 0xc0c05c30f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022c18810}, 0xc0c71f53e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00b132c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00b132c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807406 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc077520060}, 0xc0ae9d74b8, 0x1, 0xc004c270a8, 0x519bbc, 0xc08800b140, 0xc086638960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077520060}, 0x120, 0xc004c27230, 0x8800b0e0, 0x0, 0x660069, 0xc08800b140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc077520060}, 0xc08800b0e0, {{{{0x0, 0x0}, {0xc09ed7f440, 0x119}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077520060}, 0x6352410, 0x25ab71e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc077520060}, {0xc08800afc0, {0x1743011056fc004a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc077520060}, 0xc0615279a0, 0xc004c279f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077520060}, 0x660089, {0xc08800afc0, {0x1743011056fc004a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc077520060}, 0xc090bb1a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011056fc004a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011056fc004a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020c4def0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020c4def0}, {0xc004c293f8, 0x660089}, 0xc004c29400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020c4def0}, {0x0}, 0xc090bb1800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020c4de60}, 0xc090bb1800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}, 0xc088ac11e0, 0xc02990f608) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}, 0x3, 0xc097b55d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004c29980}}}, {0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}, 0xc088ac11e0, 0xc097b55d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020c4de60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020c4de60}, {0x40, 0xc004c29b70}, 0xc004c29ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020c4de60}, {0x4e9d7a0, 0xc090bb1800}, 0xc088ac11e0, 0xc02990f608) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020c4de60}, 0xc010511080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc085d54360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc085d54360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795417 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe990f8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc044c71980, 0xc00adb8000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc044c71980, {0xc00adb8000, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc044c71980, {0xc00adb8000, 0x739, 0xc051417900}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0204d35e0, {0xc00adb8000, 0x7eda79, 0xc00adb8005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc08f667800, {0xc00adb8000, 0x4bd14e, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc044b4b4e8, {0xc00adb8000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc083c78978, {0x631b900, 0xc044b4b4e8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc083c78700, {0x7fe42ab59310, 0xc08f667800}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc083c78700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc083c78700, {0xc04d137000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03ab401b0, {0xc04d137000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04a1c4b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04a1c4b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04a1c50e0, {0x634aac0, 0xc04a1c4b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04a1c4a80, 0xc022b81180, 0xc027db542b, 0xc027db542d, {0x635bec0, 0xc0385a3cb0}, 0xc000d6ae00, 0xc045bafa08, 0xc027db5429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04a1c4a80, {0x63c85b8, 0xc07a3ed140}, 0xc022b81170, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09d784043, 0xb}}, 0x0, 0xc07a3ed0e0, 0xc07a3ed110, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc07a3ed140}, {0x643d810, 0xc08f667800}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07279fb00}, {0x643d810, 0xc08f667800}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07279fb00}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811806 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bc67180, {0x63c85b8, 0xc0592d5aa0}, 0xc01dc61db8, 0x1, 0x7080181831e15e, 0x37e11d600, 0xc04565cba0, 0xc01b975800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0592d5aa0}, 0x120, 0xc0341cf230, 0x4565cb40, 0x0, 0x660069, 0xc04565cba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bc67180, {0x63c85b8, 0xc0592d5aa0}, 0xc04565cb40, {{{{0x0, 0x0}, {0xc0b646fb00, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0592d5aa0}, 0x6352410, 0x65f8a088) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bc68870, {0x63c85b8, 0xc0592d5aa0}, {0xc04565ca20, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bc68870, {0x63c85b8, 0xc0592d5aa0}, 0xc04e916dc0, 0xc0341cf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0592d5aa0}, 0x660089, {0xc04565ca20, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bf93c00, {0x63c85b8, 0xc0592d5aa0}, 0xc045d30d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592d59e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592d59e0}, {0xc0341d13f8, 0x660089}, 0xc0341d1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592d59e0}, {0x7fe3fb6a9898}, 0xc045d30b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592d5950}, 0xc045d30b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}, 0xc00a67bc20, 0xc01dc61d88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}, 0x3, 0xc062041f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0341d1980}}}, {0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}, 0xc00a67bc20, 0xc062041f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0592d5950}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592d5950}, {0x40, 0xc0341d1b70}, 0xc0341d1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0592d5950}, {0x4e9d7a0, 0xc045d30b00}, 0xc00a67bc20, 0xc01dc61d88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0592d5950}, 0xc023a4ae40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04565c480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04565c480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219141850 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc038d20570}, 0xc056471b30, {0xc01d5263d0, 0x10}, 0xc073ac1740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc038d20570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807741 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6000, {0x63c85b8, 0xc04e542450}, 0xc01576fa10, 0x1, 0xc006b470a8, 0x519bbc, 0xc06b27a720, 0xc04dd67140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04e542450}, 0x120, 0xc006b47230, 0x6b27a6c0, 0x0, 0x660069, 0xc06b27a720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6000, {0x63c85b8, 0xc04e542450}, 0xc06b27a6c0, {{{{0x0, 0x0}, {0xc0154f66c0, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04e542450}, 0x63526b0, 0x5591dc60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddb30, {0x63c85b8, 0xc04e542450}, {0xc06b27a360, {0x1743011059888b33, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc04e542450}, 0xc043f12be0, 0xc006b479f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e542450}, 0x660089, {0xc06b27a360, {0x1743011059888b33, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc04e542450}, 0xc05b046500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011059888b33, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011059888b33, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e5423c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e5423c0}, {0xc006b493f8, 0x660089}, 0xc006b49400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e5423c0}, {0x3fefffff7865836b}, 0xc05b046300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e542330}, 0xc05b046300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}, 0xc01c5184c0, 0xc01ce24720) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}, 0x3, 0xc077e45040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006b49980}}}, {0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}, 0xc01c5184c0, 0xc077e45040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e542330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e542330}, {0x40, 0xc006b49b70}, 0xc006b49ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e542330}, {0x4e9d7a0, 0xc05b046300}, 0xc01c5184c0, 0xc01ce24720) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e542330}, 0xc03c13d2c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06b27a120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06b27a120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807056 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035cc0, {0x63c85b8, 0xc06fe6f020}, 0xc089ff3740, 0x1, 0xc00362f0a8, 0x519bbc, 0xc08413c240, 0xc071b48d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0190217a0, {0x63c85b8, 0xc06fe6f020}, 0xc08413c240, 0xc00362f1a8, 0x41, 0x60, 0x0, 0xc08413c240, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035cc0, {0x63c85b8, 0xc06fe6f020}, 0xc08413c000, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0730a2c60, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06fe6f020}, 0x63521a0, 0x1b928f10) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd9f0, {0x63c85b8, 0xc06fe6f020}, {0xc07c1410e0, {0x174301100281140c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd9f0, {0x63c85b8, 0xc06fe6f020}, 0xc0397de820, 0xc00362f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06fe6f020}, 0x660089, {0xc07c1410e0, {0x174301100281140c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e097800, {0x63c85b8, 0xc06fe6f020}, 0xc08d81ec00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100281140c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100281140c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039393c20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039393c20}, {0xc0036313f8, 0x660089}, 0xc003631400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039393c20}, {0xc0015221e0}, 0xc08d514f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039393a70}, 0xc08d514f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}, 0xc06d002060, 0xc02d7e30c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}, 0x3, 0xc0594ae440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003631980}}}, {0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}, 0xc06d002060, 0xc0594ae440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc039393a70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039393a70}, {0x40, 0xc003631b70}, 0xc003631ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc039393a70}, {0x4e9d7a0, 0xc08d514f00}, 0xc06d002060, 0xc02d7e30c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc039393a70}, 0xc051a1cba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07c1407e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07c1407e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810903 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018383180, {0x63c85b8, 0xc044d035f0}, 0xc053ea1218, 0x1, 0xc013be30a8, 0x519bbc, 0xc04a4894a0, 0xc053e503e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc044d035f0}, 0x120, 0xc013be3230, 0x4a489440, 0x0, 0x660069, 0xc04a4894a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018383180, {0x63c85b8, 0xc044d035f0}, 0xc04a489440, {{{{0x0, 0x0}, {0xc0cd8fcc60, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc044d035f0}, 0x6352230, 0x3594af18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0181df9f0, {0x63c85b8, 0xc044d035f0}, {0xc04a489320, {0x1743011070aeacf8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0181df9f0, {0x63c85b8, 0xc044d035f0}, 0xc068b2b180, 0xc013be39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044d035f0}, 0x660089, {0xc04a489320, {0x1743011070aeacf8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0184f6400, {0x63c85b8, 0xc044d035f0}, 0xc067220100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011070aeacf8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011070aeacf8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044d034a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044d034a0}, {0xc013be53f8, 0x660089}, 0xc013be5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044d034a0}, {0x7fe429382d50}, 0xc0507a1f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044d033e0}, 0xc0507a1f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}, 0xc04693df80, 0xc0d093ae28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}, 0x3, 0xc0ce40ec00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc013be5980}}}, {0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}, 0xc04693df80, 0xc0ce40ec00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044d033e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044d033e0}, {0x40, 0xc013be5b70}, 0xc013be5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044d033e0}, {0x4e9d7a0, 0xc0507a1f00}, 0xc04693df80, 0xc0d093ae28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044d033e0}, 0xc0324d7a40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04a4890e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04a4890e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812542 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074aa90, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc0857fd8c0, 0x0, 0xffff, 0xc07cbc23f0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07b68b340, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc054653880, 0xc07b68b340, {0xc0523f50e0, 0x27, 0x60}, {0xc021f21320, 0x26, 0x60}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07b68b340, {0xc0523f50e0, 0x27, 0x60}, {0xc021f21320, 0x26, 0x60}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52280, 0x26, 0xc0872581e0, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0xc0175b3590, 0xc0872582f8, 0x18, 0xc0872581f8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc087258190, 0xc05468fe40, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc087258190, {0xc04ec43e50, 0x27, 0x4a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:400 +0x97 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0872571c0, {0xc04ec43e50, 0x60, 0x4a}, 0x26, 0x27) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc0872571c0, {0xc04ec43e50, 0xc04ec43e50, 0xc021f21320}, 0x4a) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc087256f00, {0xc04ec43e50, 0x27, 0x4a}, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/pebble.(*Iterator).SeekGE(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:865 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06769b2d8, {{0xc03a992810, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:290 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05075ee00, {{0xc03a992810, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc04a8d6c00, {{0xc03a992810, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc04a8d6c00, {0x63c85b8, 0xc04bf653e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc03a992840, 0x25, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.MVCCScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc03a992840, 0x25, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2504 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc026662e70}, {{0x17430110881a517c, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:62 +0x41b github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110881a517c, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04aa85800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a5e7b67c, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01d853400, {0x63c85b8, 0xc04bf653e0}, 0xc04aa85800, 0xc052bbaaa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d853400, {0x63c85b8, 0xc04bf653e0}, 0xc04aa85800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110881a517c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110881a517c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bf65350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf65350}, {0xc03eb553f8, 0x660089}, 0xc03eb55400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bf65350}, {0x7fe137b0f488}, 0xc04aa85600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bf652c0}, 0xc04aa85600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}, 0xc0ccf7f800, 0xc0381617a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}, 0x3, 0xc02a52fa00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03eb55980}}}, {0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}, 0xc0ccf7f800, 0xc02a52fa00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bf652c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf652c0}, {0x40, 0xc03eb55b70}, 0xc03eb55ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bf652c0}, {0x4e9d7a0, 0xc04aa85600}, 0xc0ccf7f800, 0xc0381617a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bf652c0}, 0xc05c2fb080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cddc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cddc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28212 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc003179b18) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01cc2c630) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01cc2c630, {0x63c8510, 0xc017d657c0}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x63c85b8, 0xc02c253ec0}, {0xc003179c98, 0x2, 0x4000000000000000}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:193 +0x91 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00141adc0, {0x63c85b8, 0xc02c253ec0}, 0xc02c253b30) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:291 +0x5de github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x63c85b8, 0xc02c253dd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:215 +0x339 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791322 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0632c89c0}, 0xc024600cb0, {0x646cc48, 0xc0632c8a20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0632c89c0}, 0xedb7a834e, 0xc076eeb200) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0632c89c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc086296ae0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc024600ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc086296ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28319 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*incomingStream).Run(0xc0056ce000, {0x63c85b8, 0xc01cbb52f0}, 0xc000d3ab40, {0x643def0, 0xc0147980b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:342 +0x1cb github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Receiver).PushUpdates(0xc01876a6c0, {0x643def0, 0xc0147980b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/receiver.go:89 +0x134 github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb._SideTransport_PushUpdates_Handler({0x4cd47a0, 0xc01876a6c0}, {0x6400fe0, 0xc02dc18d40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/bazel-out/k8-opt/bin/pkg/kv/kvserver/closedts/ctpb/ctpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb/service.pb.go:346 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}, 0xc0002e73f8, 0x51c46f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}, 0x3, 0xc01e9a89c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0xc02e0acfc0}}}, {0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}, 0xc0002e73f8, 0xc01e9a89c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01cbb5260}, {0x460ef4, 0xc01f7f5bc8}, 0xc01f7f5b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}, 0xc0002e73f8, 0xc01e9a89c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4cd47a0, 0xc01876a6c0}, {0x6401850, 0xc0009fcea0}, 0xc0002e73f8, 0x51c46f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc00cf4f380}, 0xc00b38ad80, 0xc007b7b7a0, 0x9181540, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc00cf4f380}, 0xc00b38ad80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806253 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc07a6e5ce0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28119 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc00c88de60, {0x6dd7, 0xc00d506780}) github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:145 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219176710 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc057b8c390, 0x161) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc044044a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0440450e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc055adb300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc055adb300, {0x63c8510, 0xc057b8c3c0}, 0xc091301dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc084daad70) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc057b8c3c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc084daad70) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809018 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc047147350}, 0xc02ca18138, 0x1, 0xc00b7170a8, 0x519bbc, 0xc0500f42a0, 0xc07acbc720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc047147350}, 0x120, 0xc00b717230, 0x500f4240, 0x0, 0x660069, 0xc0500f42a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc047147350}, 0xc0500f4240, {{{{0x0, 0x0}, {0xc0208d66c0, 0x27}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047147350}, 0x6352410, 0x290b5c48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc047147350}, {0xc0500f4120, {0x17430110617803ba, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc047147350}, 0xc02eeab360, 0xc00b7179f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047147350}, 0x660089, {0xc0500f4120, {0x17430110617803ba, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc047147350}, 0xc0b684b300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110617803ba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110617803ba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0471472c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0471472c0}, {0xc00b7193f8, 0x660089}, 0xc00b719400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0471472c0}, {0xc0a1ffc29090cc08}, 0xc0b684b000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047147230}, 0xc0b684b000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}, 0xc044164ea0, 0xc03d493818) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}, 0x3, 0xc05b727a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b719980}}}, {0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}, 0xc044164ea0, 0xc05b727a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047147230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047147230}, {0x40, 0xc00b719b70}, 0xc00b719ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047147230}, {0x4e9d7a0, 0xc0b684b000}, 0xc044164ea0, 0xc03d493818) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047147230}, 0xc026b2a5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0500f4000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0500f4000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812186 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc031dafdd0}, 0xc05940fe18, 0x1, 0x7080181d7a3b37, 0x37e11d600, 0xc06f45d260, 0xc06f289b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc031dafdd0}, 0x120, 0xc039d1b230, 0x6f45d200, 0x0, 0x660069, 0xc06f45d260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc031dafdd0}, 0xc06f45d200, {{{{0x0, 0x0}, {0xc08f7a4900, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc031dafdd0}, 0x6352410, 0x42012c60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc031dafdd0}, {0xc06f45d0e0, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc031dafdd0}, 0xc05ad73040, 0xc039d1b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc031dafdd0}, 0x660089, {0xc06f45d0e0, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc031dafdd0}, 0xc073560c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031dafd40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031dafd40}, {0xc039d1d3f8, 0x660089}, 0xc039d1d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031dafd40}, {0xc0015221e0}, 0xc073560a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031dafcb0}, 0xc073560a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}, 0xc02dda6f40, 0xc05940fde8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}, 0x3, 0xc03b76f380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc039d1d980}}}, {0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}, 0xc02dda6f40, 0xc03b76f380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc031dafcb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031dafcb0}, {0x40, 0xc039d1db70}, 0xc039d1dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc031dafcb0}, {0x4e9d7a0, 0xc073560a00}, 0xc02dda6f40, 0xc05940fde8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc031dafcb0}, 0xc020225920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f310e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f310e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801294 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0b2c00, {0x63c85b8, 0xc01620d890}, 0xc0274b0200, 0xc075833180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc01620d890}, 0xc0274b0200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff50afd18, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff50afd18, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01620d7d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01620d7d0}, {0xc0129f93f8, 0x660089}, 0xc0129f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01620d7d0}, {0x33}, 0xc0274b0000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01620d6b0}, 0xc0274b0000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}, 0xc0824b8820, 0xc03ca5c600) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}, 0x3, 0xc051597040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0129f9980}}}, {0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}, 0xc0824b8820, 0xc051597040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01620d6b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01620d6b0}, {0x40, 0xc0129f9b70}, 0xc0129f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01620d6b0}, {0x4e9d7a0, 0xc0274b0000}, 0xc0824b8820, 0xc03ca5c600) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01620d6b0}, 0xc028d5e960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc057e170e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc057e170e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803522 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc075bd8720}, 0xc044154018, 0x1, 0xc0762f1ff0, 0x519bbc, 0xc048268060, 0xc046f7b500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc075bd8720}, 0x1e0, 0xc0762f2178, 0x48268000, 0x0, 0x660069, 0xc048268060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc075bd8720}, 0xc048268000, {{{{0x0, 0x0}, {0xc08414c240, 0xdf}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc075bd8720}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc075bd8720}, {0xc03a90a480, {0x174301102cffdd45, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc075bd8720}, 0xc022b45cc0, 0xc08bf48938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc075bd8720}, 0x660089, {0xc03a90a480, {0x174301102cffdd45, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc075bd8720}, 0xc00d050c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cffdd45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cffdd45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075bd8690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075bd8690}, {0xc08bf4a340, 0x660089}, 0xc08bf4a348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075bd8690}, {0xc00102e4e0}, 0xc00d050b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc075bd8630}, 0xc00d050b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02ba20600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812578 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811807 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc04d65aba0}, 0xc0a7bf6c30, 0x1, 0xc03510b0a8, 0x519bbc, 0xc012160a80, 0xc0364ee600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04d65aba0}, 0x120, 0xc03510b230, 0x12160a20, 0x0, 0x660069, 0xc012160a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc04d65aba0}, 0xc012160a20, {{{{0x0, 0x0}, {0xc09ff56fc0, 0x25}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d65aba0}, 0x6352410, 0x204d20f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc04d65aba0}, {0xc0121605a0, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc04d65aba0}, 0xc088618c80, 0xc03510b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d65aba0}, 0x660089, {0xc0121605a0, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc04d65aba0}, 0xc081d58400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d65aae0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d65aae0}, {0xc03510d3f8, 0x660089}, 0xc03510d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d65aae0}, {0xc0015221e0}, 0xc081d58200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d65a930}, 0xc081d58200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}, 0xc04be86fa0, 0xc04fe46c48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}, 0x3, 0xc0601fe0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03510d980}}}, {0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}, 0xc04be86fa0, 0xc0601fe0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d65a930}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d65a930}, {0x40, 0xc03510db70}, 0xc03510dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d65a930}, {0x4e9d7a0, 0xc081d58200}, 0xc04be86fa0, 0xc04fe46c48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d65a930}, 0xc04ba608a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04565c5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04565c5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799749 [runnable]: sync.runtime_SemacquireMutex(0xc094dc2690, 0x8f, 0x7fdec588aa00) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc094dc2538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc094dc2560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc059cbcd80, {0xffffffffffffff15, 0xc094dc26c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc059cbcd80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc059cbcd80, {0xc05fc90120, 0x32, 0x90}, {0xc07bd1db90, 0x32, 0x86}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc03d70de98, {0xc05fc90120, 0x32, 0x90}, {0xc07bd1db90, 0x32, 0x86}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc03d70c7c0, {0xc07bd1db90, 0x86, 0x90}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc03d70c7c0, {0xc05fc90120, 0x86, 0xc07bd1db90}, {0xc07bd1db90, 0xc049f04750, 0x31}, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc03d70c500, {0xc07bd1db90, 0x32, 0x86}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0593e7328, {{0xc049f04750, 0x31, 0xd0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0728cb200, {{0xc03f0ccae0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07d87f000, {0x63c85b8, 0xc03a74d020}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03a74d020}, {0x64a6560, 0xc0728cb200}, {0xc03f0ccae0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03a74d020}, {0x6466ed8, 0xc0593e7080}, {0xc03f0ccae0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0747147e0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0a843cc00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0205cf400, {0x63c85b8, 0xc03a74d020}, 0xc0a843cc00, 0xc068611c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cf400, {0x63c85b8, 0xc03a74d020}, 0xc0a843cc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03a74cf60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a74cf60}, {0xc0032fa8f0, 0x660089}, 0xc0032fa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03a74cf60}, {0xc00102e4e0}, 0xc0a843cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a74ced0}, 0xc0a843cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc09ed741e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810005 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc016c4d000, {0x63c85b8, 0xc0302a1ef0}, 0xc057b94200, 0xc075272aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc0302a1ef0}, 0xc057b94200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102df8528a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102df8528a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0302a1e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0302a1e60}, {0xc028aaf3f8, 0x660089}, 0xc028aaf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0302a1e60}, {0x0}, 0xc057b94000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0302a1dd0}, 0xc057b94000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}, 0xc08dc22460, 0xc05b1269d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}, 0x3, 0xc042c1fd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028aaf980}}}, {0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}, 0xc08dc22460, 0xc042c1fd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0302a1dd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0302a1dd0}, {0x40, 0xc028aafb70}, 0xc028aafab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0302a1dd0}, {0x4e9d7a0, 0xc057b94000}, 0xc08dc22460, 0xc05b1269d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0302a1dd0}, 0xc0317f8ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc005489560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc005489560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806530 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc088d4d410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28617 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0003df278, 0xfc618) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xadd280) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc0003df260, {0x63c8510, 0xc0304720c0}, 0x327c1) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xde github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x63c85b8, 0xc02cc735c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x357 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809990 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc0594df770}, 0xc0411c8258, 0x1, 0x70801805da0b06, 0x37e11d600, 0xc023227020, 0xc04374c960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0594df770}, 0x120, 0xc02873d230, 0x23226fc0, 0x0, 0x660069, 0xc023227020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc0594df770}, 0xc023226fc0, {{{{0x0, 0x0}, {0xc08dfe1440, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0594df770}, 0x6352410, 0x29b7bfd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc0594df770}, {0xc023226b40, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc0594df770}, 0xc052ea6460, 0xc02873d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0594df770}, 0x660089, {0xc023226b40, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0594df770}, 0xc084dc6600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0594df6e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0594df6e0}, {0xc02873f3f8, 0x660089}, 0xc02873f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0594df6e0}, {0x4b4653565a784e30}, 0xc084dc6400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0594df590}, 0xc084dc6400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}, 0xc032ba5220, 0xc0411c8228) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}, 0x3, 0xc05b30cf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02873f980}}}, {0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}, 0xc032ba5220, 0xc05b30cf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0594df590}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0594df590}, {0x40, 0xc02873fb70}, 0xc02873fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0594df590}, {0x4e9d7a0, 0xc084dc6400}, 0xc032ba5220, 0xc0411c8228) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0594df590}, 0xc020577e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c5d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c5d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28276 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2({0x63c85b8, 0xc02ce14a80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:251 +0x1c5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802566 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc074374548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0743745b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc092e66cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc044d43180, {0xc04f09acc0, 0x2b, 0x2b}, {0xc04f09ac90, 0x2b, 0x2b}, 0x1, 0x6c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092e66b88, {0xc04f09acc0, 0x1, 0x2b}, {0xc04f09ac90, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092e66b88, {0xc04f09acc0, 0x2b, 0x2b}, {0xc04f09ac90, 0x2b, 0x2b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092e662c0, {0xc04f09ac90, 0x2b, 0x2b}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092e662c0, {0xc04f09acc0, 0x2b, 0xc04f09ac90}, {0xc04f09ac90, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092e66000, {0xc04f09ac90, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05feacc08, {{0xc03ab44870, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc057870700, {{0xc03ab44870, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc049923400, {0x63c85b8, 0xc018fa0060}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc018fa0060}, {0x64a6560, 0xc057870700}, {0xc03ab44870, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc018fa0060}, {0x6466ed8, 0xc05feacc00}, {0xc03ab44870, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0a63e8be8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04d6f7d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f1d800, {0x63c85b8, 0xc018fa0060}, 0xc04d6f7d00, 0xc03b364320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f1d800, {0x63c85b8, 0xc018fa0060}, 0xc04d6f7d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08a925fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a925fb0}, {0xc001c988f0, 0x660089}, 0xc001c988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08a925fb0}, {0xc00102e4e0}, 0xc04d6f7c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08a925f50}, 0xc04d6f7c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220810}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809797 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0122d5800, {0x63c85b8, 0xc032c3c8d0}, 0xc0917c6a00, 0xc070e26fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0122d5800, {0x63c85b8, 0xc032c3c8d0}, 0xc0917c6a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066f30438, 0x0, 0x0}, 0xc024087160, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066f30438, 0x0, 0x0}, 0xc024087160, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc032c3c810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc032c3c810}, {0xc01478d3f8, 0x660089}, 0xc01478d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc032c3c810}, {0x7fe3b6782a78}, 0xc0917c6700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc032c3c780}, 0xc0917c6700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}, 0xc03c5909c0, 0xc04f2bf1b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}, 0x3, 0xc015147280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01478d980}}}, {0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}, 0xc03c5909c0, 0xc015147280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc032c3c780}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc032c3c780}, {0x40, 0xc01478db70}, 0xc01478dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc032c3c780}, {0x4e9d7a0, 0xc0917c6700}, 0xc03c5909c0, 0xc04f2bf1b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc032c3c780}, 0xc036131380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07abd3440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07abd3440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795877 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0329e35c0}, 0xc02457ccb0, {0x646cc48, 0xc0329e3620}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0329e35c0}, 0xedb7a834e, 0xc012ea4ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0329e35c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0bc19a870}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc02457cef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0bc19a870}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802826 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa3c0, {0x63c85b8, 0xc038d37200}, 0xc04f9e8360, 0x1, 0xc063e150a8, 0x519bbc, 0xc0467004e0, 0xc059e46de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc038d37200}, 0x120, 0xc063e15230, 0x46700480, 0x0, 0x660069, 0xc0467004e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa3c0, {0x63c85b8, 0xc038d37200}, 0xc046700480, {{{{0x0, 0x0}, {0xc079b039e0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc038d37200}, 0x6352410, 0x41bc4498) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701180, {0x63c85b8, 0xc038d37200}, {0xc046700360, {0x174301100f7498e6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701180, {0x63c85b8, 0xc038d37200}, 0xc042351c20, 0xc063e159f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc038d37200}, 0x660089, {0xc046700360, {0x174301100f7498e6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edca800, {0x63c85b8, 0xc038d37200}, 0xc0a9490700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100f7498e6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100f7498e6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc038d37110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc038d37110}, {0xc063e173f8, 0x660089}, 0xc063e17400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc038d37110}, {0xc0015221e0}, 0xc0a9490500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc038d37080}, 0xc0a9490500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}, 0xc0724e6080, 0xc05c80e558) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}, 0x3, 0xc033747bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc063e17980}}}, {0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}, 0xc0724e6080, 0xc033747bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc038d37080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc038d37080}, {0x40, 0xc063e17b70}, 0xc063e17ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc038d37080}, {0x4e9d7a0, 0xc0a9490500}, 0xc0724e6080, 0xc05c80e558) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc038d37080}, 0xc07527f5c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc036210480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc036210480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 216823226 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc05f204e90, 0x605) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc059cb2000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc059cb2660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc013999900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc013999900, {0x63c8510, 0xc05f204ec0}, 0xc0643f5dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0470d0290) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc05f204ec0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0470d0290) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809381 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021e1c140, {0x63c85b8, 0xc030bde7e0}, 0xc045221758, 0x1, 0xc021b150a8, 0x519bbc, 0xc0c8e836e0, 0xc0610cf020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc030bde7e0}, 0x120, 0xc021b15230, 0xc8e83680, 0x0, 0x660069, 0xc0c8e836e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021e1c140, {0x63c85b8, 0xc030bde7e0}, 0xc0c8e83680, {{{{0x0, 0x0}, {0xc0aca88fc0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc030bde7e0}, 0x6352410, 0x300d0ab8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021ef0280, {0x63c85b8, 0xc030bde7e0}, {0xc0c8e83560, {0x1743011063c6baff, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021ef0280, {0x63c85b8, 0xc030bde7e0}, 0xc0489d9860, 0xc021b159f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc030bde7e0}, 0x660089, {0xc0c8e83560, {0x1743011063c6baff, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021efb000, {0x63c85b8, 0xc030bde7e0}, 0xc060ccb900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063c6baff, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063c6baff, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc030bde720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030bde720}, {0xc021b173f8, 0x660089}, 0xc021b17400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc030bde720}, {0x7fe32e7624a0}, 0xc060ccb700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc030bde690}, 0xc060ccb700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}, 0xc0671ad580, 0xc03178f950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}, 0x3, 0xc0b902b680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc021b17980}}}, {0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}, 0xc0671ad580, 0xc0b902b680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc030bde690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030bde690}, {0x40, 0xc021b17b70}, 0xc021b17ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc030bde690}, {0x4e9d7a0, 0xc060ccb700}, 0xc0671ad580, 0xc03178f950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc030bde690}, 0xc072e10420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0c8e83440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0c8e83440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812787 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28233 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x63c85b8, 0xc02cc8b9b0}) github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:268 +0x32e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 30501 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00c358380, {0x63c8510, 0xc01352ae80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:310 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00c358380, {0x63c85b8, 0xc0318a0f60}, 0xc02c41eff0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:335 +0xbb github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x63c85b8, 0xc0318a0f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811643 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e237540, {0x63c85b8, 0xc0580a52f0}, 0xc03ed20f18, 0x1, 0x70801816c4b1c2, 0x37e11d600, 0xc065ff3800, 0xc05f37cba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0580a52f0}, 0x120, 0xc029879230, 0x65ff37a0, 0x0, 0x660069, 0xc065ff3800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e237540, {0x63c85b8, 0xc0580a52f0}, 0xc065ff37a0, {{{{0x0, 0x0}, {0xc0429145a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0580a52f0}, 0x6352410, 0x3824cdd8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00a63d270, {0x63c85b8, 0xc0580a52f0}, {0xc065ff3680, {0x174301106f6cccce, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00a63d270, {0x63c85b8, 0xc0580a52f0}, 0xc0225cfea0, 0xc0298799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0580a52f0}, 0x660089, {0xc065ff3680, {0x174301106f6cccce, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001912c00, {0x63c85b8, 0xc0580a52f0}, 0xc06fd2ee00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f6cccce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f6cccce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0580a5260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0580a5260}, {0xc02987b3f8, 0x660089}, 0xc02987b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0580a5260}, {0x7fe422b1b798}, 0xc06fd2ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0580a51d0}, 0xc06fd2ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}, 0xc06b015760, 0xc03ed20ee8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}, 0x3, 0xc01637f140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02987b980}}}, {0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}, 0xc06b015760, 0xc01637f140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0580a51d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0580a51d0}, {0x40, 0xc02987bb70}, 0xc02987bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0580a51d0}, {0x4e9d7a0, 0xc06fd2ec00}, 0xc06b015760, 0xc03ed20ee8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0580a51d0}, 0xc03690db60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc065ff3560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc065ff3560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805098 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02336d540, {0x63c85b8, 0xc0592a1500}, 0xc05b94e678, 0x1, 0x0, 0x1, 0xc07124fda0, 0xc037897c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0592a1500}, 0x120, 0xc089ea0728, 0x7124fd40, 0xc00239aea0, 0x18, 0xc07124fda0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02336d540, {0x63c85b8, 0xc0592a1500}, 0xc07124fd40, {{{{0x0, 0x0}, {0xc04652d9e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0592a1500}, 0x6352410, 0x71c3e950) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02336ab90, {0x63c85b8, 0xc0592a1500}, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02336ab90, {0x63c85b8, 0xc0592a1500}, 0xc0195d23c0, 0xc076d8cee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0592a1500}, 0x660089, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023c70800, {0x63c85b8, 0xc0592a1500}, 0xc06173b700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592a1470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592a1470}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592a1470}, {0xc00102e4e0}, 0xc06173b600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592a1410}, 0xc06173b600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fcdbc20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799940 [runnable]: syscall.Syscall6(0x11, 0x343, 0x7fe17cf59020, 0x24a7, 0x232dc84, 0x0, 0x0) GOROOT/src/syscall/asm_linux_amd64.s:43 +0x5 syscall.Pread(0x7e28, {0x7fe17cf59020, 0x118564e, 0xc00074b630}, 0x0) GOROOT/src/syscall/zsyscall_linux_amd64.go:1212 +0x58 internal/poll.(*FD).Pread(0xc00239b040, {0x7fe17cf59020, 0x4ba25b, 0x24a7}, 0x200000003) GOROOT/src/internal/poll/fd_unix.go:193 +0xf9 os.(*File).pread(...) GOROOT/src/os/file_posix.go:41 os.(*File).ReadAt(0xc0570c87f0, {0x7fe17cf59020, 0xc0847723c0, 0x1184134}, 0x24c7) GOROOT/src/os/file.go:137 +0x191 github.com/cockroachdb/pebble/vfs.enospcFile.ReadAt(...) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_full.go:361 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0359298c0, {0x232dc84, 0x24a2}, 0x0, 0xc0aeed0b78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2339 +0x3f2 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0aeed0900, {0xffffffffffffff15, 0xc0847725c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0aeed0900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0aeed0900, {0xc005e41a40, 0x2b, 0x6d}, 0x13, 0xb3) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0aeed0900, {0xc015cfdc00, 0x2b, 0x6d}, {0xc005e41a40, 0x51c9860, 0xc005e41a40}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0aeed0900, {0xc015cfdc00, 0x2b, 0x74}, {0xc005e41a40, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07aa56140, {0xc015cfdc00, 0x2b, 0x74}, {0xc005e41a40, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07aa54cc0, {0xc005e41a40, 0x6d, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07aa54cc0, {0xc015cfdc00, 0x2b, 0xc005e41a40}, {0xc005e41a40, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07aa54a00, {0xc005e41a40, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05c90d088, {{0xc0160b6900, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0626e8100, {{0xc0160b6900, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f93b800, {0x63c85b8, 0xc06bdb4420}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06bdb4420}, {0x64a6560, 0xc0626e8100}, {0xc0160b6900, 0x20eda, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06bdb4420}, {0x6466ed8, 0xc05c90d080}, {0xc0160b6900, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02f14d548}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc02756ab00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209bd000, {0x63c85b8, 0xc06bdb4420}, 0xc02756ab00, 0xc05b113860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209bd000, {0x63c85b8, 0xc06bdb4420}, 0xc02756ab00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06bdb4390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06bdb4390}, {0xc0936988f0, 0x660089}, 0xc0936988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06bdb4390}, {0xc00102e4e0}, 0xc02756a300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06bdb4330}, 0xc02756a300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01174b830}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803626 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc079925d70}, 0xc056cc7938, 0x1, 0xc051f11ff0, 0x519bbc, 0xc042f3ede0, 0xc042bb34a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc079925d70}, 0x120, 0xc051f12178, 0x42f3ed80, 0x0, 0x660069, 0xc042f3ede0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc079925d70}, 0xc042f3ed80, {{{{0x0, 0x0}, {0xc048c1ab40, 0x51}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc079925d70}, 0x6352410, 0x3080ff90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc079925d70}, {0xc042f3ec60, {0x174301102e67b889, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc079925d70}, 0xc064b03a40, 0xc058194938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc079925d70}, 0x660089, {0xc042f3ec60, {0x174301102e67b889, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc079925d70}, 0xc076906f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e67b889, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e67b889, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079925ce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079925ce0}, {0xc058196340, 0x660089}, 0xc058196348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079925ce0}, {0xc00102e4e0}, 0xc076906e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079925c80}, 0xc076906e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc04ff92660}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 15948 [select]: github.com/cockroachdb/cockroach/pkg/rpc.respStreamClientAdapter.recvInternal({{0x63c85b8, 0xc023181ef0}, 0xc01878f980, 0xc01878faa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:694 +0xbf github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:723 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc00167cf00, {0x63c8510, 0xc002936240}, {{0xc01373f7b0, 0x2, 0x8}, {0xc01373f7b8, 0x2, 0x8}}, {0x1742d9fd9cb8d278, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:519 +0xd84 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc00167cf00, {0x63c8510, 0xc002936240}, 0xc01878f860, {{0xc01373f7b0, 0x2, 0x8}, {0xc01373f7b8, 0x2, 0x8}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:336 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed.func1.1({0x63c8510, 0xc002936240}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:120 +0xbe github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 209248945 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b24f0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0649f9700, 0xc00b834800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0649f9700, {0xc00b834800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0649f9700, {0xc00b834800, 0x5, 0xc09276abc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc038434ef8, {0xc00b834800, 0x80, 0x7f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0221df6e0, {0xc00b834800, 0xc078edac60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc020318138, {0xc00b834800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc057e3f3f8, {0x631b900, 0xc020318138}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc057e3f180, {0x7fe42ab59310, 0xc0221df6e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc057e3f180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc057e3f180, {0xc032e87000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0853c7128, {0xc032e87000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03d71eb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03d71eb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03d71f0e0, {0x634aac0, 0xc03d71eb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03d71ea80, 0xc031710e20, 0xc09276b42b, 0xc09276b42d, {0x635bec0, 0xc029944bd0}, 0xc000d6ae00, 0xc036712768, 0xc09276b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03d71ea80, {0x63c85b8, 0xc044279260}, 0xc031710e00, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a7280042, 0x8}}, 0x0, 0xc044279200, 0xc044279230, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc044279260}, {0x643d810, 0xc0221df6e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc075a4ade0}, {0x643d810, 0xc0221df6e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc075a4ade0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790661 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03005fb80, {0x63c85b8, 0xc03067c2a0}, 0xc04521ed98, 0x1, 0xc092b2e388, 0x519bbc, 0xc07a700300, 0xc0722f73e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc022685320, {0x63c85b8, 0xc03067c2a0}, 0xc022685360, 0xc092b2e488, 0x31, 0x40, 0xffffffffffffffff, 0xc07a700300, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03005fb80, {0x63c85b8, 0xc03067c2a0}, 0xc07a700240, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0629587e0, 0x2}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03067c2a0}, 0x63521a0, 0x4f455410) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030007a40, {0x63c85b8, 0xc03067c2a0}, {0xc07a700120, {0x1743011002c20ba8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030007a40, {0x63c85b8, 0xc03067c2a0}, 0xc07471af00, 0xc092b2ecd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03067c2a0}, 0x660089, {0xc07a700120, {0x1743011002c20ba8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0300a9c00, {0x63c85b8, 0xc03067c2a0}, 0xc08e597000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011002c20ba8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011002c20ba8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03067c120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03067c120}, {0xc092b306d8, 0x660089}, 0xc092b306e0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03067c120}, {0xc00102e4e0}, 0xc08e596f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064359f50}, 0xc08e596f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc01ebfe790, {0x63c85b8, 0xc064358540}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc077570000, {0x63c85b8, 0xc064358540}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc077570000, {0x63c8510, 0xc0440a5e80}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc077570000, {0x63c8510, 0xc0440a5e80}, 0xc07a2d7a30, {0x7fe42b790cf0, 0xc03949e6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc077570000, {0x63c85b8, 0xc05904fad0}, {{0x63f88f8, 0xc07a2d7a30}, {0xc0832cd7ca, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc077570000, {0x63c85b8, 0xc05904fad0}, {{0x63f88f8, 0xc07a2d7a30}, {0xc0832cd7ca, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc07a2d7a30}, {0xc0832cd7ca, 0x23}, 0x0, 0x0}, {0x3b58518e, 0xedb7a834f, 0x0}, {0x3b5858e0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc077570000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc077570000, {0x63c8510, 0xc062038b80}, 0xc070137dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc016087d70) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc062038b80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc016087d70) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802552 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc023ad2400, {0x63c85b8, 0xc02081dc50}, 0xc046fdc000, 0xc022d91400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023ad2400, {0x63c85b8, 0xc02081dc50}, 0xc046fdc000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025505477, 0x0, 0x0}, 0xc03060afa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025505477, 0x0, 0x0}, 0xc03060afa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02081d7a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02081d7a0}, {0xc051f153f8, 0x660089}, 0xc051f15400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02081d7a0}, {0x7fe34e25be60}, 0xc0115e5f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02081d5f0}, 0xc0115e5f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}, 0xc0968d05c0, 0xc06316fa70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}, 0x3, 0xc056d17240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc051f15980}}}, {0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}, 0xc0968d05c0, 0xc056d17240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02081d5f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02081d5f0}, {0x40, 0xc051f15b70}, 0xc051f15ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02081d5f0}, {0x4e9d7a0, 0xc0115e5f00}, 0xc0968d05c0, 0xc06316fa70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02081d5f0}, 0xc05afddd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0248befc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0248befc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811695 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc03b7b2720}, 0xc04a423b48, 0x1, 0xc039d836d8, 0x519bbc, 0xc06e8920c0, 0xc08839c660) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01cfd2e00, {0x63c85b8, 0xc03b7b2720}, 0xc05c9a98c0, 0xc039d837d8, 0x6e8920c0, 0x0, 0x0, 0xc06e8920c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc03b7b2720}, 0xc06e892000, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0412c26c0, 0x4b}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03b7b2720}, 0x6352260, 0x36ce8190) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc03b7b2720}, {0xc0229c4120, {0x1743011061c0e40f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc03b7b2720}, 0xc01858bd60, 0xc039d84020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03b7b2720}, 0x660089, {0xc0229c4120, {0x1743011061c0e40f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc03b7b2720}, 0xc04fb2d200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061c0e40f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061c0e40f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03b7b2690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b7b2690}, {0xc06318ba28, 0x660089}, 0xc06318ba30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03b7b2690}, {0xc00102e4e0}, 0xc04fb2d100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03b7b2600}, 0xc04fb2d100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc06242d350, {0x63c85b8, 0xc03b7b2570}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc06242d350, {0x63c85b8, 0xc03b7b2570}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc03b7b2570}, {0x4f67ad2, 0x10beec0}, 0xc01f3f0fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219805024 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc02956d440}, 0xc0269d7590, 0x1, 0x708017dc20d00f, 0x37e11d600, 0xc058aa4a80, 0xc07207b4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02956d440}, 0x120, 0xc053473230, 0x58aa4a20, 0xc0027f8000, 0x18, 0xc058aa4a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc02956d440}, 0xc058aa4a20, {{{{0x0, 0x0}, {0xc0582e2d80, 0x22}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02956d440}, 0x6352410, 0x421b1f20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc02956d440}, {0xc058aa4900, {0x174301103f3ec9da, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc02956d440}, 0xc0132352c0, 0xc0534739f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02956d440}, 0x660089, {0xc058aa4900, {0x174301103f3ec9da, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc02956d440}, 0xc067ff5d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f3ec9da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f3ec9da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02956d350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02956d350}, {0xc0534753f8, 0x660089}, 0xc053475400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02956d350}, {0xc066e3e9c0}, 0xc067ff5b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02956d2c0}, 0xc067ff5b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}, 0xc06612afa0, 0xc0269d7560) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}, 0x3, 0xc06db17340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc053475980}}}, {0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}, 0xc06612afa0, 0xc06db17340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02956d2c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02956d2c0}, {0x40, 0xc053475b70}, 0xc053475ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02956d2c0}, {0x4e9d7a0, 0xc067ff5b00}, 0xc06612afa0, 0xc0269d7560) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02956d2c0}, 0xc0275c8f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058aa47e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058aa47e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808675 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaffb3740, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc058fc5b80, 0xc011abc000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc058fc5b80, {0xc011abc000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc058fc5b80, {0xc011abc000, 0x735, 0xc06da33540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06b244da0, {0xc011abc000, 0x7eda79, 0xc011abc005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03fcc8660, {0xc011abc000, 0xc0764f6c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04a6d9ed8, {0xc011abc000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc050156278, {0x631b900, 0xc04a6d9ed8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc050156000, {0x7fe42ab59310, 0xc03fcc8660}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc050156000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc050156000, {0xc075206000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc05f255e78, {0xc075206000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0edd20088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0edd20088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0edd20660, {0x634aac0, 0xc0edd20088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0edd20000, 0xc06c5d5450, 0xc0764f742b, 0xc0764f742d, {0x635bec0, 0xc04a92a750}, 0xc000d6ae00, 0xc02dc61df8, 0xc0764f7429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0edd20000, {0x63c85b8, 0xc03aafb440}, 0xc06c5d5440, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06f24b045, 0x8}}, 0x0, 0xc03aafb380, 0xc03aafb3b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03aafb440}, {0x643d810, 0xc03fcc8660}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04d8f3b90}, {0x643d810, 0xc03fcc8660}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04d8f3b90}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28287 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0013c58c0, {0x63c8510, 0xc017d653c0}, 0xc02cafa060, 0xc01db6f080) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:331 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x63c8510, 0xc017d653c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:288 +0x38 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 219721078 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe79d68, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06306c180, 0xc0718cc800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06306c180, {0xc0718cc800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06306c180, {0xc0718cc800, 0x739, 0xc050625b80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc057085f80, {0xc0718cc800, 0x7eda79, 0xc0718cc805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc068621aa0, {0xc0718cc800, 0x2, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02f753f20, {0xc0718cc800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06eaf6978, {0x631b900, 0xc02f753f20}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06eaf6700, {0x7fe42ab59310, 0xc068621aa0}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06eaf6700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06eaf6700, {0xc0aad92000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc037c06e40, {0xc0aad92000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc056515588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc056515588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc056515b60, {0x634aac0, 0xc056515588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc056515500, 0xc05b943600, 0xc004e4942b, 0xc004e4942d, {0x635bec0, 0xc0a8029050}, 0xc000d6ae00, 0xc01b7c8198, 0xc004e49429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc056515500, {0x63c85b8, 0xc0289ef7a0}, 0xc05b9435f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0ae31c043, 0xb}}, 0x0, 0xc0289ef740, 0xc0289ef770, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0289ef7a0}, {0x643d810, 0xc068621aa0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0299c9e90}, {0x643d810, 0xc068621aa0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0299c9e90}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801223 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0280dc0f0}, 0xc022330100, 0xc06440a140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0280dc0f0}, 0xc022330100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ebd1d6a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ebd1d6a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0280dc030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0280dc030}, {0xc0973747f8, 0x660089}, 0xc097374800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0280dc030}, {0xc00102e4e0}, 0xc076191f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0401b7fb0}, 0xc076191f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc024648b00, {0x63c85b8, 0xc0401b7e60}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc08a428600, {0x63c85b8, 0xc0401b7e60}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc08a428600, {0x63c8510, 0xc04d06e0c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc08a428600, {0x63c8510, 0xc04d06e0c0}, {0x63f7db8, 0x9bf6f48}, 0xc097379088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc08a428600, {0x63c85b8, 0xc044e09a70}, {{0x63f7db8, 0x9bf6f48}, {0xc0b348c896, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc08a428600, {0x63c85b8, 0xc044e09a70}, {{0x63f7db8, 0x9bf6f48}, {0xc0b348c896, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0b348c896, 0x6}, 0x0, 0x0}, {0x46798fe, 0xedb7a834f, 0x0}, {0x467a2f6, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08a428600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08a428600, {0x63c8510, 0xc013f3b300}, 0xc073097dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0836355c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc013f3b300}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0836355c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219807007 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0643581b0}, 0xc025eb2a68, 0x1, 0xc002ebd0a8, 0x519bbc, 0xc052067e00, 0xc048e71e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc08efa2a20, {0x63c85b8, 0xc0643581b0}, 0xc01e83b320, 0xc002ebd1a8, 0x52067e00, 0x0, 0x0, 0xc052067e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0643581b0}, 0xc052067d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0656a0900, 0x60}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0643581b0}, 0x6352260, 0x4f4553d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0643581b0}, {0xc052067c20, {0x174301101aa4b463, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0643581b0}, 0xc076d76780, 0xc002ebd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0643581b0}, 0x660089, {0xc052067c20, {0x174301101aa4b463, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0643581b0}, 0xc08e596d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101aa4b463, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101aa4b463, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064358120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064358120}, {0xc002ebf3f8, 0x660089}, 0xc002ebf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064358120}, {0xc0015221e0}, 0xc08e596b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064358090}, 0xc08e596b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}, 0xc06c8c63a0, 0xc016c71cc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}, 0x3, 0xc0440a5dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002ebf980}}}, {0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}, 0xc06c8c63a0, 0xc0440a5dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc064358090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064358090}, {0x40, 0xc002ebfb70}, 0xc002ebfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc064358090}, {0x4e9d7a0, 0xc08e596b00}, 0xc06c8c63a0, 0xc016c71cc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc064358090}, 0xc03f90acc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052067b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052067b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 212779337 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3e0310, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05ef79600, 0xc090f4e800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05ef79600, {0xc090f4e800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05ef79600, {0xc090f4e800, 0x735, 0xc058593680}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc04d762768, {0xc090f4e800, 0x7eda79, 0xc090f4e805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc049474720, {0xc090f4e800, 0xc08d652c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a903d8, {0xc090f4e800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0a84fa278, {0x631b900, 0xc043a903d8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0a84fa000, {0x7fe42ab59310, 0xc049474720}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0a84fa000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0a84fa000, {0xc04f194000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06c3dfae8, {0xc04f194000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06893eb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06893eb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06893f0e0, {0x634aac0, 0xc06893eb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06893ea80, 0xc03251aae0, 0xc0711bf42b, 0xc0711bf42d, {0x635bec0, 0xc007ecaab0}, 0xc000d6ae00, 0xc01f8abce8, 0xc0711bf429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06893ea80, {0x63c85b8, 0xc08a0099b0}, 0xc03251aad0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc04f190034, 0x8}}, 0x0, 0xc08a0097d0, 0xc08a009800, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc08a0099b0}, {0x643d810, 0xc049474720}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc074051860}, {0x643d810, 0xc049474720}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc074051860}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801052 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc063310ea0}, 0xc03fdb9700, 0xc0cd42b220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc063310ea0}, 0xc03fdb9700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff84f1186, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff84f1186, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc063310e10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063310e10}, {0xc00dd9d3f8, 0x660089}, 0xc00dd9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc063310e10}, {0x7fe3f1e4ff50}, 0xc03fdb9400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc063310d50}, 0xc03fdb9400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}, 0xc06c54c240, 0xc0210f6108) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}, 0x3, 0xc03b9d14c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00dd9d980}}}, {0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}, 0xc06c54c240, 0xc03b9d14c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc063310d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063310d50}, {0x40, 0xc00dd9db70}, 0xc00dd9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc063310d50}, {0x4e9d7a0, 0xc03fdb9400}, 0xc06c54c240, 0xc0210f6108) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc063310d50}, 0xc0509338c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04b1ee6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04b1ee6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805192 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc04870df20}, 0xc064a6a900, 0xc02a1a3ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04870df20}, 0xc064a6a900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f3ec9da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f3ec9da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04870de90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04870de90}, {0xc07b8c13f8, 0x660089}, 0xc07b8c1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04870de90}, {0x7fe3e0ed1868}, 0xc064a6a600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04870de00}, 0xc064a6a600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}, 0xc05c42ce00, 0xc03f476a08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}, 0x3, 0xc05c333a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b8c1980}}}, {0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}, 0xc05c42ce00, 0xc05c333a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04870de00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04870de00}, {0x40, 0xc07b8c1b70}, 0xc07b8c1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}, 0x203017, 0x203017) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04870de00}, {0x4e9d7a0, 0xc064a6a600}, 0xc05c42ce00, 0xc03f476a08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04870de00}, 0xc00df5f800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc072108a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc072108a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218980033 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc041142810, 0x4) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc020b19500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc020b19b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc09a07c600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc09a07c600, {0x63c8510, 0xc041142840}, 0xc0a8e9bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04727e2b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc041142840}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04727e2b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809362 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc058fc0180}, 0xc032a1ec00, 0xc066746c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc058fc0180}, 0xc032a1ec00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110640a93f0, 0x0, 0x0}, 0xc056bfe3c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110640a93f0, 0x0, 0x0}, 0xc056bfe3c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc058fc00f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058fc00f0}, {0xc01f4d13f8, 0x660089}, 0xc01f4d1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc058fc00f0}, {0xc0015221e0}, 0xc032a1ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc058fc0060}, 0xc032a1ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}, 0xc02fa34fc0, 0xc048ff9f20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}, 0x3, 0xc0525ccb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f4d1980}}}, {0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}, 0xc02fa34fc0, 0xc0525ccb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc058fc0060}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058fc0060}, {0x40, 0xc01f4d1b70}, 0xc01f4d1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc058fc0060}, {0x4e9d7a0, 0xc032a1ea00}, 0xc02fa34fc0, 0xc048ff9f20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc058fc0060}, 0xc0228527e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679da5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679da5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812215 [semacquire]: sync.runtime_SemacquireMutex(0xc0027f96c0, 0x3, 0xc0027f96c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc03990ae40) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fde34ed5000, 0x7e83, 0x7fd58bde9f40) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc0156d0480, 0x7fde34ed5001, 0x7fde34ed5020, 0x2fa6, 0x2fab) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0177622c0, {0x375ce97, 0x2fa6}, 0x0, 0xc04d42f8f8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04d42f680, {0xffffffffffffff15, 0xc03990b0c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc04d42f680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc04d42f680, {0xc0051192d0, 0x2b, 0x6b}, 0x13, 0x2f) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04d42f680, {0xc0c60bdf00, 0x2b, 0x6b}, {0xc0051192d0, 0x51c9860, 0xc0051192d0}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04d42f680, {0xc0c60bdf00, 0x2b, 0x74}, {0xc0051192d0, 0x2b, 0x6b}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091ecde98, {0xc0c60bdf00, 0x2b, 0x74}, {0xc0051192d0, 0x2b, 0x6b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091ecc7c0, {0xc0051192d0, 0x6b, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091ecc7c0, {0xc0c60bdf00, 0x2b, 0xc0051192d0}, {0xc0051192d0, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091ecc500, {0xc0051192d0, 0x2b, 0x6b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc01e640c08, {{0xc06342e8a0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc046c99e00, {{0xc06342e8a0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0777f7400, {0x63c85b8, 0xc030191ad0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc030191ad0}, {0x64a6560, 0xc046c99e00}, {0xc06342e8a0, 0x0, 0x30}, {0x174301107e48800a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc030191ad0}, {0x6466ed8, 0xc01e640c00}, {0xc06342e8a0, 0x2a, 0x30}, {0x174301107e48800a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc054220b28}, {{0x174301107e48800a, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301107e48800a, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc08256a100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174301109c15e50a, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01de0d000, {0x63c85b8, 0xc030191ad0}, 0xc08256a100, 0xc0753de500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01de0d000, {0x63c85b8, 0xc030191ad0}, 0xc08256a100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e48800a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e48800a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc030191a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030191a40}, {0xc0399113f8, 0x660089}, 0xc039911400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc030191a40}, {0x7fe13dc790f8}, 0xc048c45e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc030191980}, 0xc048c45e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}, 0xc01e517920, 0xc0c05c2cd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}, 0x3, 0xc03d2c8d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc039911980}}}, {0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}, 0xc01e517920, 0xc03d2c8d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc030191980}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030191980}, {0x40, 0xc039911b70}, 0xc039911ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc030191980}, {0x4e9d7a0, 0xc048c45e00}, 0xc01e517920, 0xc0c05c2cd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc030191980}, 0xc0c71f4de0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0143977a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0143977a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806336 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc088fd7350}, 0xc08d4a8960, 0x1, 0xc00c56d0a8, 0x519bbc, 0xc05aa5ef00, 0xc04c36fa40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc088fd7350}, 0x120, 0xc00c56d230, 0x5aa5eea0, 0x0, 0x660069, 0xc05aa5ef00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc088fd7350}, 0xc05aa5eea0, {{{{0x0, 0x0}, {0xc052ccd8c0, 0x44}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc088fd7350}, 0x6352410, 0xf06388) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc088fd7350}, {0xc05aa5ea20, {0x174301104d82da47, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc088fd7350}, 0xc07d3cc320, 0xc00c56d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc088fd7350}, 0x660089, {0xc05aa5ea20, {0x174301104d82da47, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc088fd7350}, 0xc032b60d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d82da47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d82da47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc088fd72c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc088fd72c0}, {0xc00c56f3f8, 0x660089}, 0xc00c56f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc088fd72c0}, {0xc0015221e0}, 0xc032b60b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc088fd7230}, 0xc032b60b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}, 0xc08dae2540, 0xc0745a0600) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}, 0x3, 0xc0847068c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00c56f980}}}, {0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}, 0xc08dae2540, 0xc0847068c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc088fd7230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc088fd7230}, {0x40, 0xc00c56fb70}, 0xc00c56fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc088fd7230}, {0x4e9d7a0, 0xc032b60b00}, 0xc08dae2540, 0xc0745a0600) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc088fd7230}, 0xc017a54d80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05aa5e900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05aa5e900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812003 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f7c3000, {0x63c85b8, 0xc06199ca80}, 0xc060cc9e00, 0xc06c4ff900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f7c3000, {0x63c85b8, 0xc06199ca80}, 0xc060cc9e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107c36161b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107c36161b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06199c9f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06199c9f0}, {0xc0346cd3f8, 0x660089}, 0xc0346cd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06199c9f0}, {0xc0015221e0}, 0xc060cc9b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06199c960}, 0xc060cc9b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}, 0xc0262c6740, 0xc013d6cc78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}, 0x3, 0xc04e09e740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0346cd980}}}, {0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}, 0xc0262c6740, 0xc04e09e740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06199c960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06199c960}, {0x40, 0xc0346cdb70}, 0xc0346cdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06199c960}, {0x4e9d7a0, 0xc060cc9b00}, 0xc0262c6740, 0xc013d6cc78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06199c960}, 0xc063da7680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc065f5c900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc065f5c900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 214268390 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc03c2614d0, 0x166d) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc03d370a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc03d3710e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc029540c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc029540c00, {0x63c8510, 0xc03c261540}, 0xc0549cbdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc076fcdf20) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03c261540}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc076fcdf20) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810135 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc076482750}, {0xc0aedd02c0, 0x2, 0xc031b39e58}, {0x0, {0x1743010ff5156ffd, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc076482750}, 0x18701, 0xc0588dfc20, 0x20, 0xc020362260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc076482750}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801065 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x80, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0a20c9180, 0xc0453f5c28) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0a20c9180, {0xc04b03c1c0, 0x1f, 0x6e}, {0xc0701c63c0, 0x1f, 0x30}, 0x0, 0x20) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091721f88, {0xc04b03c1c0, 0x1, 0x30}, {0xc0701c63c0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091721f88, {0xc04b03c1c0, 0x1f, 0x6e}, {0xc0701c63c0, 0x1f, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0917216c0, {0xc0701c63c0, 0xc0453f69e0, 0x11a3c29}, 0x7fe429c0e8c0, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0917216c0, {0xc04b03c1c0, 0x1f, 0x30}, {0xc0701c63c0, 0x1e, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091721400, {0xc0701c63c0, 0x1f, 0x30}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc002785b88, {{0xc032acca60, 0x1e, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04016f000, {0x63c85b8, 0xc029d87ce0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc029d87ce0}, {0x64a6618, 0xc002785b88}, {0xc032acca60, 0x0, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc029d87ce0}, {0x6466ed8, 0xc002785b80}, {0xc032acca60, 0x1e, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x63c85b8, 0xc029d87ce0}, {0x6466ed8, 0xc002785b80}, {0xc032acca60, 0xc0453f6ac8, 0x461247}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:617 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0, {0x63c85b8, 0xc029d87ce0}, {0x6466ed8, 0xc002785b80}, {0x6c, 0xa1, 0x42, 0xec, 0x32, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:98 +0xdb github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0x6c, 0xa1, 0x42, 0xec, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:1958 +0xbc github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc014024600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:201 +0x37e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110390a597f, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc016515000, {0x63c85b8, 0xc029d87ce0}, 0xc014024600, 0xc06828d680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016515000, {0x63c85b8, 0xc029d87ce0}, 0xc014024600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b3cf47f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b3cf47f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029d87c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029d87c50}, {0xc0453fb3f8, 0x660089}, 0xc0453fb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029d87c50}, {0xc08ab1ebd0}, 0xc05134e400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029d87a70}, 0xc05134e400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}, 0xc0552d6f00, 0xc0941b2258) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}, 0x3, 0xc035f7eb40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0453fb980}}}, {0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}, 0xc0552d6f00, 0xc035f7eb40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029d87a70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029d87a70}, {0x40, 0xc0453fbb70}, 0xc0453fbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029d87a70}, {0x4e9d7a0, 0xc05134e400}, 0xc0552d6f00, 0xc0941b2258) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029d87a70}, 0xc01ff8dc80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc037ba7680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc037ba7680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811145 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021ad2c80, {0x63c85b8, 0xc06b880630}, 0xc01c978480, 0x1, 0xc0047290a8, 0x519bbc, 0xc0137c05c0, 0xc0906b9f20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0841d8fc0, {0x63c85b8, 0xc06b880630}, 0xc0841d8ff0, 0xc0047291a8, 0x30, 0x40, 0xffffffffffffffff, 0xc0137c05c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021ad2c80, {0x63c85b8, 0xc06b880630}, 0xc0137c0380, {{{{0xc06b114a20, 0x1}, {0xc0589dab40, 0x1}}, {{0xc0639cec60, 0x6}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06b880630}, 0x63521a0, 0x1978a8f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a319a0, {0x63c85b8, 0xc06b880630}, {0xc01bae5320, {0x174301102e16193a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a319a0, {0x63c85b8, 0xc06b880630}, 0xc0661317c0, 0xc0047299f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b880630}, 0x660089, {0xc01bae5320, {0x174301102e16193a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c06000, {0x63c85b8, 0xc06b880630}, 0xc06f378400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e16193a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e16193a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b880570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b880570}, {0xc00472b3f8, 0x660089}, 0xc00472b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b880570}, {0xc0015221e0}, 0xc06f378200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b880480}, 0xc06f378200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}, 0xc01ed82a80, 0xc03a388ca8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}, 0x3, 0xc047939a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00472b980}}}, {0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}, 0xc01ed82a80, 0xc047939a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b880480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b880480}, {0x40, 0xc00472bb70}, 0xc00472bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b880480}, {0x4e9d7a0, 0xc06f378200}, 0xc01ed82a80, 0xc03a388ca8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b880480}, 0xc01cd8df80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01bae5200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01bae5200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28585 [select, 716 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 219806848 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e099c00, {0x63c85b8, 0xc055563bc0}, 0xc070083c00, 0xc04151c780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099c00, {0x63c85b8, 0xc055563bc0}, 0xc070083c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c9acd56, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c9acd56, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc055563b00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc055563b00}, {0xc07dca46d8, 0x660089}, 0xc07dca46e0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc055563b00}, {0xc00102e4e0}, 0xc070083b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc055563aa0}, 0xc070083b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc04e34a6e0, {0x63c85b8, 0xc055563920}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc052d54000, {0x63c85b8, 0xc055563920}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc052d54000, {0x63c8510, 0xc06c0a87c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc052d54000, {0x63c8510, 0xc06c0a87c0}, 0xc071a339c0, {0x7fe42b790cf0, 0xc0849c66c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc052d54000, {0x63c85b8, 0xc067e0fe00}, {{0x63f88f8, 0xc071a339c0}, {0xc0452c8c6f, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc052d54000, {0x63c85b8, 0xc067e0fe00}, {{0x63f88f8, 0xc071a339c0}, {0xc0452c8c6f, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc071a339c0}, {0xc0452c8c6f, 0x23}, 0x0, 0x0}, {0x65ce54f, 0xedb7a8350, 0x0}, {0x65ceaba, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc052d54000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc052d54000, {0x63c8510, 0xc090dc7c80}, 0xc08271ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc054daa990) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc090dc7c80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc054daa990) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804684 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc07c79ef30}, 0xc050158a38, 0x1, 0xc08ebdd0a8, 0x519bbc, 0xc066b65920, 0xc058224de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07c79ef30}, 0x120, 0xc08ebdd230, 0x66b658c0, 0xc00285f040, 0x18, 0xc066b65920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc07c79ef30}, 0xc066b658c0, {{{{0x0, 0x0}, {0xc0477b4ea0, 0x19}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07c79ef30}, 0x6352410, 0x3d0ac820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc07c79ef30}, {0xc066b657a0, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc07c79ef30}, 0xc053b56460, 0xc08ebdd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07c79ef30}, 0x660089, {0xc066b657a0, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc07c79ef30}, 0xc00436ee00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c79eea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c79eea0}, {0xc08ebdf3f8, 0x660089}, 0xc08ebdf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c79eea0}, {0x88d0fb5fe4296833}, 0xc00436e900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c79ede0}, 0xc00436e900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}, 0xc0320d8b80, 0xc050159800) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}, 0x3, 0xc05ec89440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc08ebdf980}}}, {0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}, 0xc0320d8b80, 0xc05ec89440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c79ede0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c79ede0}, {0x40, 0xc08ebdfb70}, 0xc08ebdfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}, 0x203017, 0x203017) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c79ede0}, {0x4e9d7a0, 0xc00436e900}, 0xc0320d8b80, 0xc050159800) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c79ede0}, 0xc07062d740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc066b65680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc066b65680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28398 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc00a9a9800, {0x63c85b8, 0xc01c2274d0}, 0x14347, 0xc012f03c38, 0xc000d3ab40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:269 +0x53f github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x63c85b8, 0xc01c2274d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:222 +0x3c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809154 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc041e10d50}, 0xc053c6fcb0, 0x1, 0xc00c495ff0, 0x519bbc, 0xc08e80daa0, 0xc076f2c420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc041e10d50}, 0x1e0, 0xc00c496178, 0x8e80da40, 0x0, 0x660069, 0xc08e80daa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc041e10d50}, 0xc08e80da40, {{{{0x0, 0x0}, {0xc04fe16b40, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc041e10d50}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc041e10d50}, {0xc073fcc360, {0x1743011061c0e40f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc041e10d50}, 0xc061134aa0, 0xc005be6938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041e10d50}, 0x660089, {0xc073fcc360, {0x1743011061c0e40f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc041e10d50}, 0xc010dedd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061c0e40f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061c0e40f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041e10cc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041e10cc0}, {0xc005be8340, 0x660089}, 0xc005be8348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041e10cc0}, {0xc00102e4e0}, 0xc010dedc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041e10c60}, 0xc010dedc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0176d7020}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807336 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6140, {0x63c85b8, 0xc02de4fc20}, 0xc034bd7860, 0x1, 0xc0051ff0a8, 0x519bbc, 0xc02442e5c0, 0xc06f39be00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04c36e7e0, {0x63c85b8, 0xc02de4fc20}, 0xc04c36e818, 0xc0051ff1a8, 0x41, 0x60, 0xffffffffffffffff, 0xc02442e5c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6140, {0x63c85b8, 0xc02de4fc20}, 0xc02442e380, {{{{0x0, 0x0}, {0xc089988c60, 0x6}}, {{0xc0b646fc20, 0x5}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02de4fc20}, 0x63521a0, 0x25ab6df8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddbd0, {0x63c85b8, 0xc02de4fc20}, {0xc07c42e900, {0x1743011000599690, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddbd0, {0x63c85b8, 0xc02de4fc20}, 0xc03effe1e0, 0xc0051ff9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02de4fc20}, 0x660089, {0xc07c42e900, {0x1743011000599690, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099c00, {0x63c85b8, 0xc02de4fc20}, 0xc08e4f3e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011000599690, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011000599690, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02de4faa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02de4faa0}, {0xc0052013f8, 0x660089}, 0xc005201400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02de4faa0}, {0xc0015221e0}, 0xc08e4f3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02de4f890}, 0xc08e4f3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}, 0xc088ac05a0, 0xc082101fc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}, 0x3, 0xc075ce39c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005201980}}}, {0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}, 0xc088ac05a0, 0xc075ce39c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02de4f890}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02de4f890}, {0x40, 0xc005201b70}, 0xc005201ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02de4f890}, {0x4e9d7a0, 0xc08e4f3b00}, 0xc088ac05a0, 0xc082101fc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02de4f890}, 0xc03733e9c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07c42e7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07c42e7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801996 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05004cc00, {0x63c85b8, 0xc0bb638720}, 0xc027a75700, 0xc071cbd360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05004cc00, {0x63c85b8, 0xc0bb638720}, 0xc027a75700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101f4b11dc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101f4b11dc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0bb638690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0bb638690}, {0xc00eb273f8, 0x660089}, 0xc00eb27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0bb638690}, {0x7fe3b672c188}, 0xc027a75500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0bb6385d0}, 0xc027a75500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}, 0xc03c8c3a00, 0xc027eb5dd0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}, 0x3, 0xc072e66180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00eb27980}}}, {0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}, 0xc03c8c3a00, 0xc072e66180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0bb6385d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0bb6385d0}, {0x40, 0xc00eb27b70}, 0xc00eb27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0bb6385d0}, {0x4e9d7a0, 0xc027a75500}, 0xc03c8c3a00, 0xc027eb5dd0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0bb6385d0}, 0xc028743c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ee145a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ee145a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807020 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc022e1e2d0}, {0xc002179600, 0x4, 0xc09c0c2658}, {0x0, {0x1743010ff20e420b, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc022e1e2d0}, 0x1871d, 0xc043cff200, 0xcd, 0xc06c220680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc022e1e2d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28440 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc050cdc240) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0623cd810, {0x4d05620, 0xc041cd0800}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4e) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc050cdc000, 0xc0854276e0, 0xc0854276b0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc050cdc000, {0x4d05620, 0xc041cd0800}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc0469bfad0}, {0x4f49ba0, 0x2}, {0x4d054e0, 0xc01a538660}, {0x4d05620, 0xc041cd0800}, 0x4677c5, {0xc0193d9a40, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c8580, 0xc01a5386c0}, {0x4f49ba0, 0x7fe457d811d8}, {0x4d054e0, 0xc01a538660}, {0x4d05620, 0xc041cd0800}, 0x65f5fb, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0xc028da5510, {0x63c8580, 0xc01a5386c0}, {0x4f49ba0, 0x63c8580}, {0x4d054e0, 0xc01a538660}, {0x4d05620, 0xc041cd0800}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/rpc.(*heartbeatClient).Ping(0xc01978ad90, {0x63c8580, 0xc01a5386c0}, 0x33d91f5d, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/bazel-out/k8-opt/bin/pkg/rpc/rpc_go_proto_/github.com/cockroachdb/cockroach/pkg/rpc/heartbeat.pb.go:246 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3.2({0x63c8580, 0xc01a5386c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1498 +0x85 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0188a2510}, {0x4ef81cf, 0xd}, 0x165a0bc00, 0xc084ec1c20) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x63c85b8, 0xc0188a2510}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1503 +0x391 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0188a2510}, {0x0, 0xd}, 0xc084ec1de0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc0188a2510}, 0xc00b710b40, {0xc00e275b48, 0x12}, 0xc021bd7800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1471 +0x3d5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc0188a2510}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812710 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc02557bc80}, 0xc01842d518, 0x1, 0xc03e0850a8, 0x519bbc, 0xc02a23a300, 0xc045051ce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc047346980, {0x63c85b8, 0xc02557bc80}, 0xc081cef0e0, 0xc03e0851a8, 0x2a23a300, 0x0, 0x0, 0xc02a23a300, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc02557bc80}, 0xc02a23a240, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc044762d80, 0x50}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02557bc80}, 0x6352260, 0x42013448) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc02557bc80}, {0xc02a23a000, {0x174301104c6e8840, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc02557bc80}, 0xc07c888c80, 0xc03e0859f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02557bc80}, 0x660089, {0xc02a23a000, {0x174301104c6e8840, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc02557bc80}, 0xc03486e200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c6e8840, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c6e8840, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02557bb30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02557bb30}, {0xc03e0873f8, 0x660089}, 0xc03e087400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02557bb30}, {0x7fe423ec3b30}, 0xc03486e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02557b6b0}, 0xc03486e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}, 0xc00abd2720, 0xc0479199b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}, 0x3, 0xc01352ae00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03e087980}}}, {0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}, 0xc00abd2720, 0xc01352ae00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02557b6b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02557b6b0}, {0x40, 0xc03e087b70}, 0xc03e087ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02557b6b0}, {0x4e9d7a0, 0xc03486e000}, 0xc00abd2720, 0xc0479199b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02557b6b0}, 0xc03df19680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e9b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e9b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809909 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04ba9a3f0}, 0xc0653d7320, 0x1, 0xc0172bf0a8, 0x519bbc, 0xc0118c9380, 0xc06118c4e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04ba9a3f0}, 0x120, 0xc0172bf230, 0x118c9320, 0x0, 0x660069, 0xc0118c9380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04ba9a3f0}, 0xc0118c9320, {{{{0x0, 0x0}, {0xc09ed7eb40, 0xf1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ba9a3f0}, 0x6352410, 0x42a0fb48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04ba9a3f0}, {0xc0118c90e0, {0x1743011067c5825b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04ba9a3f0}, 0xc04c9a97c0, 0xc0172bf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ba9a3f0}, 0x660089, {0xc0118c90e0, {0x1743011067c5825b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04ba9a3f0}, 0xc004c40900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067c5825b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067c5825b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ba9a360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ba9a360}, {0xc0172c13f8, 0x660089}, 0xc0172c1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ba9a360}, {0x0}, 0xc004c40700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ba9a2d0}, 0xc004c40700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}, 0xc056c91a20, 0xc01b3eb4e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}, 0x3, 0xc04c170500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0172c1980}}}, {0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}, 0xc056c91a20, 0xc04c170500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ba9a2d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ba9a2d0}, {0x40, 0xc0172c1b70}, 0xc0172c1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ba9a2d0}, {0x4e9d7a0, 0xc004c40700}, 0xc056c91a20, 0xc01b3eb4e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ba9a2d0}, 0xc0302a7c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0118c8fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0118c8fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810341 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01876da40, {0x63c85b8, 0xc027a03650}, 0xc0444db260, 0x1, 0xc01a0070a8, 0x519bbc, 0xc0233984e0, 0xc057878060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc027a03650}, 0x120, 0xc01a007230, 0x23398480, 0x0, 0x660069, 0xc0233984e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01876da40, {0x63c85b8, 0xc027a03650}, 0xc023398480, {{{{0x0, 0x0}, {0xc063d079e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc027a03650}, 0x6352410, 0x412aca68) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01893e000, {0x63c85b8, 0xc027a03650}, {0xc023398240, {0x174301106bd4f57d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01893e000, {0x63c85b8, 0xc027a03650}, 0xc020fc7d60, 0xc01a0079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc027a03650}, 0x660089, {0xc023398240, {0x174301106bd4f57d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018945800, {0x63c85b8, 0xc027a03650}, 0xc01ca69d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bd4f57d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bd4f57d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027a035c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027a035c0}, {0xc01a0093f8, 0x660089}, 0xc01a009400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027a035c0}, {0x7fe3c2f0ad80}, 0xc01ca69b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027a03530}, 0xc01ca69b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}, 0xc07a403e20, 0xc04a2d53e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}, 0x3, 0xc0bdce2f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a009980}}}, {0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}, 0xc07a403e20, 0xc0bdce2f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc027a03530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027a03530}, {0x40, 0xc01a009b70}, 0xc01a009ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc027a03530}, {0x4e9d7a0, 0xc01ca69b00}, 0xc07a403e20, 0xc04a2d53e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc027a03530}, 0xc02de2c840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc023398120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc023398120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803873 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc01dafea50}, 0xc0542d6dc8, 0x1, 0x0, 0x1, 0xc055ee34a0, 0xc0522daa80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01dafea50}, 0x120, 0xc078a82728, 0x55ee3440, 0x0, 0x660069, 0xc055ee34a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc01dafea50}, 0xc055ee3440, {{{{0x0, 0x0}, {0xc083d96240, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01dafea50}, 0x6352410, 0x34330e20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc01dafea50}, {0xc055ee27e0, {0x1743011032129bc8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc01dafea50}, 0xc071c06140, 0xc076d8cee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01dafea50}, 0x660089, {0xc055ee27e0, {0x1743011032129bc8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc01dafea50}, 0xc06f9e5800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032129bc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032129bc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01dafe9c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01dafe9c0}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01dafe9c0}, {0xc00102e4e0}, 0xc06f9e5600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01dafe960}, 0xc06f9e5600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07d081ef0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812704 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28129 [select, 716 minutes]: github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1({0x63c85b8, 0xc02ca0c570}) github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1041 +0x106 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219794240 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc00d0f3050) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc00d0f30b8, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc08578c5f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0d540ee00, {0xc018c88000, 0x2b, 0x80}, {0xc013246e80, 0x2b, 0x74}, 0x1, 0xc5) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc08578c488, {0xc018c88000, 0x1, 0x74}, {0xc013246e80, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc08578c488, {0xc018c88000, 0x2b, 0x80}, {0xc013246e80, 0x2b, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc08578bbc0, {0xc013246e80, 0x74, 0x80}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc08578bbc0, {0xc018c88000, 0x2b, 0xc013246e80}, {0xc013246e80, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc08578b900, {0xc013246e80, 0x2b, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04875b608, {{0xc045418060, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc02c028400, {{0xc045418060, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0a6f4e800, {0x63c85b8, 0xc079056780}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc079056780}, {0x64a6560, 0xc02c028400}, {0xc045418060, 0xc00bc1fa40, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc079056780}, {0x6466ed8, 0xc04875b600}, {0xc045418060, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc015393170}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc074c3e800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020af3400, {0x63c85b8, 0xc079056780}, 0xc074c3e800, 0xc0509352c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020af3400, {0x63c85b8, 0xc079056780}, 0xc074c3e800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0790566f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0790566f0}, {0xc00d0f93f8, 0x660089}, 0xc00d0f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0790566f0}, {0x7fe3f3315530}, 0xc074c3e500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079056630}, 0xc074c3e500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}, 0xc04654d280, 0xc0715c9950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}, 0x3, 0xc028382240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00d0f9980}}}, {0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}, 0xc04654d280, 0xc028382240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079056630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079056630}, {0x40, 0xc00d0f9b70}, 0xc00d0f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079056630}, {0x4e9d7a0, 0xc074c3e500}, 0xc04654d280, 0xc0715c9950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079056630}, 0xc061295ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03bacf200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03bacf200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219777568 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc027c054d0, 0x65) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0610f6a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0610f70e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc094af6c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc094af6c00, {0x63c8510, 0xc027c05500}, 0xc0528a1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc061c71240) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc027c05500}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc061c71240) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219795533 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a5a4e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06139b880, 0xc008104800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06139b880, {0xc008104800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06139b880, {0xc008104800, 0x735, 0xc0586ff900}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc050632038, {0xc008104800, 0x7eda79, 0xc008104805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04bd745a0, {0xc008104800, 0x3, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a90060, {0xc008104800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc083a1e5f8, {0x631b900, 0xc043a90060}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc083a1e380, {0x7fe42ab59310, 0xc04bd745a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc083a1e380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc083a1e380, {0xc005a7f000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03aa5f3e0, {0xc005a7f000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05f6f4088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05f6f4088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05f6f4660, {0x634aac0, 0xc05f6f4088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05f6f4000, 0xc0875c1040, 0xc04bd2f42b, 0xc04bd2f42d, {0x635bec0, 0xc04b4181b0}, 0xc000d6ae00, 0xc074a5b2c8, 0xc04bd2f429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05f6f4000, {0x63c85b8, 0xc06f8c8540}, 0xc0875c1030, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09b75a03b, 0x8}}, 0x0, 0xc06f8c84e0, 0xc06f8c8510, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06f8c8540}, {0x643d810, 0xc04bd745a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01e80a360}, {0x643d810, 0xc04bd745a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01e80a360}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806906 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e04d400, {0x63c85b8, 0xc092e22ae0}, 0xc07447fa00, 0xc04ae64320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc092e22ae0}, 0xc07447fa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101740f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101740f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc092e22a50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092e22a50}, {0xc0046b93f8, 0x660089}, 0xc0046b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc092e22a50}, {0x0}, 0xc07447f800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc092e229c0}, 0xc07447f800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}, 0xc08fc488a0, 0xc038af92f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}, 0x3, 0xc09071d340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0046b9980}}}, {0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}, 0xc08fc488a0, 0xc09071d340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc092e229c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092e229c0}, {0x40, 0xc0046b9b70}, 0xc0046b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc092e229c0}, {0x4e9d7a0, 0xc07447f800}, 0xc08fc488a0, 0xc038af92f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc092e229c0}, 0xc07be2ad20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc081f8bb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc081f8bb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28127 [select]: github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc000cac3f0, {0x63c85b8, 0xc02ca0c3c0}) github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:181 +0x2ca github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812691 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 29518 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8580, 0xc018ed4300}, 0x0, 0xc0b3f9b180) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c8580, 0xc018ed4300}, 0xc00080e678) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/ts.(*DB).storeKvs(0xc007b7b1d0, {0x63c8580, 0xc018ed4300}, {0xc078f48000, 0x559, 0x1742fe48cf0bc000}) github.com/cockroachdb/cockroach/pkg/ts/db.go:318 +0x9e github.com/cockroachdb/cockroach/pkg/ts.(*DB).tryStoreData(0xc007b7b1d0, {0x63c8580, 0xc018ed4300}, 0x63c85b8, {0xc076d46000, 0x559, 0x0}) github.com/cockroachdb/cockroach/pkg/ts/db.go:256 +0xa5 github.com/cockroachdb/cockroach/pkg/ts.(*DB).StoreData(0xc007b7b1d0, {0x63c8580, 0xc018ed4300}, 0x912799, {0xc076d46000, 0xedb7a834f, 0x0}) github.com/cockroachdb/cockroach/pkg/ts/db.go:221 +0xbf github.com/cockroachdb/cockroach/pkg/ts.(*poller).poll.func1.1({0x63c8580, 0xc018ed4300}) github.com/cockroachdb/cockroach/pkg/ts/db.go:202 +0x45 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc08d3dd320}, {0x4ee0749, 0x7}, 0xdf8475800, 0xc0abc51d48) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/ts.(*poller).poll.func1({0x63c85b8, 0xc02cc8bcb0}) github.com/cockroachdb/cockroach/pkg/ts/db.go:200 +0x16e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc02cc8bcb0}, {0xc0017cad80, 0xc0017cad80}, 0xc04b20de70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/ts.(*poller).poll(0xc025280000) github.com/cockroachdb/cockroach/pkg/ts/db.go:191 +0xc5 github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1({0xc0018eef00, 0xc02d493290}) github.com/cockroachdb/cockroach/pkg/ts/db.go:175 +0x86 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 28468 [chan receive, 716 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).rangeFeedWithRangeID(0xc012374000, 0x6437410, 0xc00e4807e0, {0x7fe42b743500, 0xc00b441d20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:254 +0x845 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).RangeFeed(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_rangefeed.go:150 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).RangeFeed(0xc0082db200, 0xc00e4807e0, {0x6437410, 0xc00b441d20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:3082 +0xf7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).RangeFeed(0xc000d3ab40, 0xc00e4807e0, {0x6437410, 0xc00b441d20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:216 +0xf6 github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0x6437410, 0x63c85b8, {0x6437410, 0xc00b441d20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1227 +0x38 github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func1() github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:753 +0xeb created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:750 +0x1cf goroutine 219812620 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0304cbec0}, 0xc031cfb668, 0x1, 0x7080183b7b1bc3, 0x37e11d600, 0xc07258a9c0, 0xc0518cc9c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06c256600, {0x63c85b8, 0xc0304cbec0}, 0xc088ff2a20, 0xc04289d1a8, 0x7258a9c0, 0x0, 0x0, 0xc07258a9c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0304cbec0}, 0xc07258a900, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc02f72ed80, 0xb5}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0304cbec0}, 0x6352260, 0x47c52658) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0304cbec0}, {0xc07258a7e0, {0x174301105c286a81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0304cbec0}, 0xc073b05cc0, 0xc04289d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0304cbec0}, 0x660089, {0xc07258a7e0, {0x174301105c286a81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0304cbec0}, 0xc0460df900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c286a81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c286a81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0304cbbf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304cbbf0}, {0xc04289f3f8, 0x660089}, 0xc04289f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0304cbbf0}, {0xc0015221e0}, 0xc0460df700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0304cba10}, 0xc0460df700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}, 0xc03ebdd980, 0xc031cfb638) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}, 0x3, 0xc015b9e7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04289f980}}}, {0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}, 0xc03ebdd980, 0xc015b9e7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0304cba10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304cba10}, {0x40, 0xc04289fb70}, 0xc04289fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0304cba10}, {0x4e9d7a0, 0xc0460df700}, 0xc03ebdd980, 0xc031cfb638) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0304cba10}, 0xc01c342120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219719026 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4e1c8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0587f9c00, 0xc055c94800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0587f9c00, {0xc055c94800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0587f9c00, {0xc055c94800, 0x735, 0xc062adfb80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc052e795c8, {0xc055c94800, 0x7eda79, 0xc055c94805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc018230900, {0xc055c94800, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc023415350, {0xc055c94800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0552b5078, {0x631b900, 0xc023415350}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0552b4e00, {0x7fe42ab59310, 0xc018230900}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0552b4e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0552b4e00, {0xc066125000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04a8aaba0, {0xc066125000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc066c03588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc066c03588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc066c03b60, {0x634aac0, 0xc066c03588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc066c03500, 0xc042f18ab0, 0xc09ce5d42b, 0xc09ce5d42d, {0x635bec0, 0xc03a402510}, 0xc000d6ae00, 0xc02fc29708, 0xc09ce5d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc066c03500, {0x63c85b8, 0xc02bee5350}, 0xc042f18aa0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a310b021, 0x8}}, 0x0, 0xc02bee5290, 0xc02bee52c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02bee5350}, {0x643d810, 0xc018230900}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03a1e9980}, {0x643d810, 0xc018230900}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03a1e9980}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807857 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc011762150}, 0xc0c5889320, 0x1, 0xc00c8956d8, 0x519bbc, 0xc07748b860, 0xc0c8590300) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06f9edd40, {0x63c85b8, 0xc011762150}, 0xc033b72120, 0xc00c8957d8, 0x7748b860, 0x0, 0x0, 0xc07748b860, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc011762150}, 0xc07748b7a0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0321a8000, 0x68}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc011762150}, 0x6352260, 0x321a1790) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc011762150}, {0xc07748b680, {0x174301101ebd1d6a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc011762150}, 0xc048c06960, 0xc00c896020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc011762150}, 0x660089, {0xc07748b680, {0x174301101ebd1d6a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc011762150}, 0xc012571c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ebd1d6a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ebd1d6a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0117620c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0117620c0}, {0xc075e57a28, 0x660089}, 0xc075e57a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0117620c0}, {0x24}, 0xc012571800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0442bbf80}, 0xc012571800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc05b346850, {0x63c85b8, 0xc0442bbe90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc05b346850, {0x63c85b8, 0xc0442bbe90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0442bbe90}, {0x4f67ad2, 0x6446ff8}, 0xc002ca3fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219811146 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021ad2c80, {0x63c85b8, 0xc0350e2120}, 0xc037362078, 0x1, 0xc00671f0a8, 0x519bbc, 0xc0b65345c0, 0xc066cae0c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc03caee000, {0x63c85b8, 0xc0350e2120}, 0xc03caee040, 0xc00671f1a8, 0x30, 0x40, 0xffffffffffffffff, 0xc0b65345c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021ad2c80, {0x63c85b8, 0xc0350e2120}, 0xc0b6534380, {{{{0xc06b114a20, 0x1}, {0xc0428358c0, 0x4}}, {{0xc044e30120, 0x8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0350e2120}, 0x63521a0, 0x47a9c698) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a319a0, {0x63c85b8, 0xc0350e2120}, {0xc0498b1e60, {0x174301100bf53c6f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a319a0, {0x63c85b8, 0xc0350e2120}, 0xc052bbaa00, 0xc00671f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0350e2120}, 0x660089, {0xc0498b1e60, {0x174301100bf53c6f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c06000, {0x63c85b8, 0xc0350e2120}, 0xc06df82800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100bf53c6f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100bf53c6f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0350e2090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0350e2090}, {0xc0067213f8, 0x660089}, 0xc006721400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0350e2090}, {0x7fe3c1500428}, 0xc06df82600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044227fb0}, 0xc06df82600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}, 0xc046699cc0, 0xc04916d338) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}, 0x3, 0xc0344cbd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006721980}}}, {0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}, 0xc046699cc0, 0xc0344cbd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044227fb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044227fb0}, {0x40, 0xc006721b70}, 0xc006721ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044227fb0}, {0x4e9d7a0, 0xc06df82600}, 0xc046699cc0, 0xc04916d338) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044227fb0}, 0xc06026ca80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04c1f0900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04c1f0900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 24701 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc007b00630, {0x63c8510, 0xc008167d80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:319 +0x997 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x63c85b8, 0xc02cf44ba0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:189 +0x274 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801999 [runnable]: sync.runtime_SemacquireMutex(0xc0947c8690, 0x8f, 0x7fe16a6f4140) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0947c8538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0947c8560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0a6d34000, {0xffffffffffffff15, 0xc0947c86c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0a6d34000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0a6d34000, {0xc02bacc400, 0x32, 0x7f}, {0xc02a675400, 0x32, 0x73}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc085f71998, {0xc02bacc400, 0x32, 0x7f}, {0xc02a675400, 0x32, 0x73}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc085f702c0, {0xc02a675400, 0x73, 0x7f}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc085f702c0, {0xc02bacc400, 0x73, 0xc02a675400}, {0xc02a675400, 0xc086775760, 0x31}, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc085f70000, {0xc02a675400, 0x32, 0x73}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc074d818a8, {{0xc086775760, 0x31, 0xb0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0343cc600, {{0xc0238a1410, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc083351800, {0x63c85b8, 0xc0738536b0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0738536b0}, {0x64a6560, 0xc0343cc600}, {0xc0238a1410, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0738536b0}, {0x6466ed8, 0xc074d81600}, {0xc0238a1410, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc043c7dfe0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0d2801800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ebbc00, {0x63c85b8, 0xc0738536b0}, 0xc0d2801800, 0xc02dd4e0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ebbc00, {0x63c85b8, 0xc0738536b0}, 0xc0d2801800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073853590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073853590}, {0xc0abc4a8f0, 0x660089}, 0xc0abc4a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073853590}, {0xc00102e4e0}, 0xc0d2801700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0738534d0}, 0xc0d2801700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0bb639290}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810278 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02505ec00, {0x63c85b8, 0xc0429aee70}, 0xc05f8ab300, 0xc02405a3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02505ec00, {0x63c85b8, 0xc0429aee70}, 0xc05f8ab300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c8275df, 0x0, 0x0}, 0xc03399f6f0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c8275df, 0x0, 0x0}, 0xc03399f6f0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0429aeba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0429aeba0}, {0xc010d6f3f8, 0x660089}, 0xc010d6f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0429aeba0}, {0xc01b5bc820}, 0xc05f8aaf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0429aeb10}, 0xc05f8aaf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}, 0xc00d92b480, 0xc028acf7a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}, 0x3, 0xc043b8ccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010d6f980}}}, {0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}, 0xc00d92b480, 0xc043b8ccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0429aeb10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0429aeb10}, {0x40, 0xc010d6fb70}, 0xc010d6fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0429aeb10}, {0x4e9d7a0, 0xc05f8aaf00}, 0xc00d92b480, 0xc028acf7a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0429aeb10}, 0xc01bb88720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0638b06c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0638b06c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801965 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06bc00, {0x63c85b8, 0xc01fe64840}, 0xc0a570fa00, 0xc02bd9dc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc01fe64840}, 0xc0a570fa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffb380575, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffb380575, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fe647b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fe647b0}, {0xc0110e33f8, 0x660089}, 0xc0110e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fe647b0}, {0x0}, 0xc0a570f800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fe64690}, 0xc0a570f800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}, 0xc0621ff6a0, 0xc09d385bd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}, 0x3, 0xc00f24f380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0110e3980}}}, {0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}, 0xc0621ff6a0, 0xc00f24f380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01fe64690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fe64690}, {0x40, 0xc0110e3b70}, 0xc0110e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01fe64690}, {0x4e9d7a0, 0xc0a570f800}, 0xc0621ff6a0, 0xc09d385bd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01fe64690}, 0xc027452000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04cb0e480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04cb0e480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810173 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030065c00, {0x63c85b8, 0xc03e370630}, 0xc061e91000, 0xc0895b54a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc03e370630}, 0xc061e91000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c52aae2, 0x0, 0x0}, 0xc053007780, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c52aae2, 0x0, 0x0}, 0xc053007780, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03e3705a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03e3705a0}, {0xc0246f33f8, 0x660089}, 0xc0246f3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03e3705a0}, {0x7fdd4a3590f8}, 0xc061e90e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03e370510}, 0xc061e90e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}, 0xc02906a760, 0xc04ebf7a28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}, 0x3, 0xc05a45e000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0246f3980}}}, {0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}, 0xc02906a760, 0xc05a45e000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03e370510}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03e370510}, {0x40, 0xc0246f3b70}, 0xc0246f3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03e370510}, {0x4e9d7a0, 0xc061e90e00}, 0xc02906a760, 0xc04ebf7a28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03e370510}, 0xc0be9db080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc057879200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc057879200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803462 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc01540c090}, 0xc04881fe60, 0x1, 0xc07085b0a8, 0x519bbc, 0xc03a3f2180, 0xc03896f260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01540c090}, 0x120, 0xc07085b230, 0x3a3f2120, 0x0, 0x660069, 0xc03a3f2180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc01540c090}, 0xc03a3f2120, {{{{0x0, 0x0}, {0xc04bea2240, 0xdc}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01540c090}, 0x6352410, 0x210ac3d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc01540c090}, {0xc03a3f2000, {0x174301102ce0c9c2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc01540c090}, 0xc02a1a3f40, 0xc07085b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01540c090}, 0x660089, {0xc03a3f2000, {0x174301102ce0c9c2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01540c090}, 0xc0571e9200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ce0c9c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ce0c9c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08291df50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08291df50}, {0xc07085d3f8, 0x660089}, 0xc07085d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08291df50}, {0xa56092ac091b0192}, 0xc0571e9000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08291dec0}, 0xc0571e9000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}, 0xc078c2e020, 0xc02629cb40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}, 0x3, 0xc064298200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07085d980}}}, {0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}, 0xc078c2e020, 0xc064298200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08291dec0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08291dec0}, {0x40, 0xc07085db70}, 0xc07085dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08291dec0}, {0x4e9d7a0, 0xc0571e9000}, 0xc078c2e020, 0xc02629cb40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08291dec0}, 0xc02b684120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc037ef1e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc037ef1e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806962 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ad17900, {0x63c85b8, 0xc0931dd920}, 0xc0449d2180, 0x1, 0xc004ec10a8, 0x519bbc, 0xc048d5ecc0, 0xc070d4b800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0931dd920}, 0x120, 0xc004ec1230, 0x48d5ec60, 0x0, 0x660069, 0xc048d5ecc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ad17900, {0x63c85b8, 0xc0931dd920}, 0xc048d5ec60, {{{{0x0, 0x0}, {0xc05c3b2c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0931dd920}, 0x6352410, 0x4f454cf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ad46410, {0x63c85b8, 0xc0931dd920}, {0xc048d5eb40, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ad46410, {0x63c85b8, 0xc0931dd920}, 0xc0678be320, 0xc004ec19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0931dd920}, 0x660089, {0xc048d5eb40, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ad59c00, {0x63c85b8, 0xc0931dd920}, 0xc0791e7400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0931dd890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0931dd890}, {0xc004ec33f8, 0x660089}, 0xc004ec3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0931dd890}, {0x0}, 0xc0791e7200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0931dd800}, 0xc0791e7200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}, 0xc08fc48e60, 0xc0449d2228) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}, 0x3, 0xc0927a8ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004ec3980}}}, {0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}, 0xc08fc48e60, 0xc0927a8ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0931dd800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0931dd800}, {0x40, 0xc004ec3b70}, 0xc004ec3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0931dd800}, {0x4e9d7a0, 0xc0791e7200}, 0xc08fc48e60, 0xc0449d2228) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0931dd800}, 0xc065d03200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc000b4b320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc000b4b320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811535 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c0cf540, {0x63c85b8, 0xc051a209f0}, 0xc0433bab88, 0x1, 0x70801816837e39, 0x37e11d600, 0xc0364eecc0, 0xc05ba2e3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc051a209f0}, 0x120, 0xc034a9b230, 0x364eec60, 0x0, 0x660069, 0xc0364eecc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c0cf540, {0x63c85b8, 0xc051a209f0}, 0xc0364eec60, {{{{0x0, 0x0}, {0xc05b3745a0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc051a209f0}, 0x6352410, 0x46738128) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c0d49b0, {0x63c85b8, 0xc051a209f0}, {0xc0364eeb40, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c0d49b0, {0x63c85b8, 0xc051a209f0}, 0xc0314e0320, 0xc034a9b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc051a209f0}, 0x660089, {0xc0364eeb40, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bd95800, {0x63c85b8, 0xc051a209f0}, 0xc07ad03600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc051a20960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051a20960}, {0xc034a9d3f8, 0x660089}, 0xc034a9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc051a20960}, {0x10610000a00a}, 0xc07ad03400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc051a208d0}, 0xc07ad03400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}, 0xc028223f00, 0xc0433bab58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}, 0x3, 0xc0448034c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034a9d980}}}, {0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}, 0xc028223f00, 0xc0448034c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc051a208d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051a208d0}, {0x40, 0xc034a9db70}, 0xc034a9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc051a208d0}, {0x4e9d7a0, 0xc07ad03400}, 0xc028223f00, 0xc0433bab58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc051a208d0}, 0xc01b732480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0522aefc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0522aefc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805587 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc03d39e840}, 0xc064120438, 0x1, 0xc074eff0a8, 0x519bbc, 0xc05ec72600, 0xc05ee07140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03d39e840}, 0x120, 0xc074eff230, 0x5ec725a0, 0xc002489040, 0x18, 0xc05ec72600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc03d39e840}, 0xc05ec725a0, {{{{0x0, 0x0}, {0xc020738fc0, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03d39e840}, 0x6352410, 0x20bbae18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc03d39e840}, {0xc05ec72480, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc03d39e840}, 0xc0472d15e0, 0xc074eff9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03d39e840}, 0x660089, {0xc05ec72480, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc03d39e840}, 0xc056da9800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03d39e7b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d39e7b0}, {0xc074f013f8, 0x660089}, 0xc074f01400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03d39e7b0}, {0x7fe3fb65c290}, 0xc056da9600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03d39e720}, 0xc056da9600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}, 0xc0724bca60, 0xc0641214b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}, 0x3, 0xc0165c5fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc074f01980}}}, {0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}, 0xc0724bca60, 0xc0165c5fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03d39e720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d39e720}, {0x40, 0xc074f01b70}, 0xc074f01ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}, 0x203005, 0x203005) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03d39e720}, {0x4e9d7a0, 0xc056da9600}, 0xc0724bca60, 0xc0641214b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03d39e720}, 0xc032cd44e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ef2a000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ef2a000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803059 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02532b400, {0x63c85b8, 0xc0780e3a70}, 0xc077ebc700, 0xc0779f7180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0780e3a70}, 0xc077ebc700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011028e75ba7, 0x0, 0x0}, 0xc03ba58050, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011028e75ba7, 0x0, 0x0}, 0xc03ba58050, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0780e39e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0780e39e0}, {0xc0677cf3f8, 0x660089}, 0xc0677cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0780e39e0}, {0x3c}, 0xc077ebc500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0780e3950}, 0xc077ebc500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}, 0xc02ea8b020, 0xc0285422d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}, 0x3, 0xc06eef13c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0677cf980}}}, {0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}, 0xc02ea8b020, 0xc06eef13c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0780e3950}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0780e3950}, {0x40, 0xc0677cfb70}, 0xc0677cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0780e3950}, {0x4e9d7a0, 0xc077ebc500}, 0xc02ea8b020, 0xc0285422d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0780e3950}, 0xc0296e7920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0562e2c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0562e2c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 28147 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc006755cf8) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc02c63d410) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc02c63d410, {0x63c8510, 0xc007b0fa80}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00167cf00, {0x63c8510, 0xc007b0fa40}, {0xc02ca0c4b0, 0x1, 0xc02ca0c3f0}, {0xc001480420, 0x7b4e4e0, 0xc0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:139 +0x625 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc02ca0c510, {0x63c8510, 0xc007b0fa40}, {0xc02ca0c4b0, 0x63c85b8, 0xc02ca0c510}, {0x7077fe423e232a8, 0x63c85b8, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:78 +0x45 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x63c8510, 0xc007b0fa40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:285 +0x7a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:169 +0x25 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:54 +0x92 goroutine 24702 [semacquire, 716 minutes]: sync.runtime_Semacquire(0x10c3ad2) GOROOT/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc00250fb18) GOROOT/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc01c879bf0) golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:40 +0x27 github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01c879bf0, {0x63c8510, 0xc01e61bac0}}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:145 +0x4a github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x63c85b8, 0xc02cf44db0}, {0xc00250fc98, 0x2, 0x4000000000000000}) github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:193 +0x91 github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00145d600, {0x63c85b8, 0xc02cf44db0}, 0xc02cf44bd0) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:291 +0x5de github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x63c85b8, 0xc02cf44d20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:215 +0x339 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812516 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219733104 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 219342975 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc03ee10840, {0x63c85b8, 0xc04e55aa50}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc047b0b300, {0x63c85b8, 0xc04e55aa50}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc047b0b300, {0x63c8510, 0xc011d07d00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc047b0b300, {0x63c8510, 0xc011d07d00}, {0x63f7db8, 0x9bf6f48}, 0xc085d77088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc047b0b300, {0x63c85b8, 0xc04037edb0}, {{0x63f7db8, 0x9bf6f48}, {0xc08b574202, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc047b0b300, {0x63c85b8, 0xc04037edb0}, {{0x63f7db8, 0x9bf6f48}, {0xc08b574202, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc08b574202, 0x6}, 0x0, 0x0}, {0x167da21, 0xedb7a834f, 0x0}, {0x167e0e6, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc047b0b300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc047b0b300, {0x63c8510, 0xc075ec9880}, 0xc0613efdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0436f4850) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc075ec9880}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0436f4850) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219794413 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02fd948a0}, 0xc0387fdcb0, {0x646cc48, 0xc02fd94900}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02fd948a0}, 0xedb7a834e, 0xc00d4afe60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02fd948a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc043190de0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0387fdef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc043190de0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812356 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc02895b620}, 0xc076cddae8, 0x1, 0x7080181d773a69, 0x37e11d600, 0xc077bd34a0, 0xc00dfeade0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02895b620}, 0x120, 0xc03482d230, 0x77bd3440, 0x0, 0x660069, 0xc077bd34a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc02895b620}, 0xc077bd3440, {{{{0x0, 0x0}, {0xc06f5fd200, 0x34}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02895b620}, 0x6352410, 0x6b067028) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc02895b620}, {0xc077bd3320, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc02895b620}, 0xc024ef9040, 0xc03482d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02895b620}, 0x660089, {0xc077bd3320, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc02895b620}, 0xc02751d700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02895b590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02895b590}, {0xc03482f3f8, 0x660089}, 0xc03482f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02895b590}, {0x4031333233584d}, 0xc02751d500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02895b320}, 0xc02751d500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}, 0xc031208720, 0xc076cddab8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}, 0x3, 0xc045b50cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03482f980}}}, {0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}, 0xc031208720, 0xc045b50cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02895b320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02895b320}, {0x40, 0xc03482fb70}, 0xc03482fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02895b320}, {0x4e9d7a0, 0xc02751d500}, 0xc031208720, 0xc076cddab8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02895b320}, 0xc042686180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0140ac120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0140ac120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812426 [semacquire]: sync.runtime_SemacquireMutex(0xc0573bdb60, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc0460eccd0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc0573bdb60, 0x42, 0x60, 0xc0460eccd0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x4d866e, 0x9185600}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).init(0xc050300480, 0xc00e0b2580, {0xc0573bdb60, 0x42, 0x60}, {0xc0460eccd0, 0x43, 0x50}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1229 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00e0b2580, {0xc0573bdb60, 0x42, 0x60}, {0xc0460eccd0, 0x43, 0x50}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2157 +0xdb github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a528c0, 0x43, 0xc0935932e8, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0xc06290a1c0, 0xc093593400, 0x0, 0xc093593300) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc093593298, 0xc0734106e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc093593298, {0xc01194f9f0, 0x42, 0x42}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:400 +0x97 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc093591bc0, {0xc01194f9f0, 0x50, 0x42}, 0x43, 0x42) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc093591bc0, {0xc01194f9f0, 0xc01194f9f0, 0xc0460eccd0}, 0x42) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc093591900, {0xc01194f9f0, 0x42, 0x42}, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/pebble.(*Iterator).SeekGE(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:865 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0584c2d78, {{0xc08f0c3920, 0x41, 0x60}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:290 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc056063700, {{0xc08f0c3920, 0x41, 0x60}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x64a6560, 0xc056063700}, {{0xc08f0c3920, 0x41, 0x60}, {0x0, 0x0, 0x0}}, 0x0, 0xc037ec4680) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:856 +0xbf github.com/cockroachdb/cockroach/pkg/storage.MVCCGarbageCollect({0x63c85b8, 0xc04f569b90}, {0x7fe42e217a50, 0xc05ed95800}, 0xc018b84080, {0xc04f569c50, 0xc086734900, 0x1}, {0x174301109e41ef2d, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:3491 +0x59a github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.GC({_, _}, {_, _}, {{0x64d2498, 0xc02e06c800}, {{0x174301109e41ef2d, 0x0, 0x0}, 0xc033eafa40, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_gc.go:115 +0x305 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109e41ef2d, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:493 +0x287 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01f6a2500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110bc0f542d, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:644 +0x190 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110bc0f542d, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:612 +0x2da github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, {{0x17430110bc0f542d, 0x0, 0x0}, {0x174301109e41ef2d, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:428 +0x5f3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc02e06c800, {0x63c85b8, 0xc04f569b90}, {0xc033eafa38, 0x8}, 0xc01f6a2500, {{0x17430110bc0f542d, 0x0, 0x0}, {0x174301109e41ef2d, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:673 +0x1c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x174301109e41ef2d, {0x63c85b8, _}, {_, _}, _, {{{0x1742da059b8e62a6, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:773 +0xaa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc02e06c800, {0x63c85b8, _}, _, _, {{{0x1742da059b8e62a6, 0x0, 0x0}, 0x0, {0x1f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:113 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06c800, {0x63c85b8, 0xc04f569b90}, 0xc01f6a2500, 0xc0648d1f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:167 +0x7b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc04f569b90}, 0xc01f6a2500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109e41ef2d, 0x0, 0x0}, 0xc033eafa40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109e41ef2d, 0x0, 0x0}, 0xc033eafa40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f569b00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f569b00}, {0xc07d93f918, 0x660089}, 0xc07d93f920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f569b00}, {0xc00102e4e0}, 0xc01f6a2300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f569aa0}, 0xc01f6a2300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06123b8c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04f569410}, {0xc00dfd36b0, 0x24}, 0x4608de038, 0xc02790dd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04f569410}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804806 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a1b3680, {0x63c85b8, 0xc04404de30}, 0xc01ec016e0, 0x1, 0x708017d61dcc80, 0x37e11d600, 0xc0528c6ba0, 0xc035a2e420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04404de30}, 0x1e0, 0xc09613b230, 0x528c6b40, 0xc006698340, 0x18, 0xc0528c6ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a1b3680, {0x63c85b8, 0xc04404de30}, 0xc0528c6b40, {{{{0x0, 0x0}, {0xc07ca017a0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x28, {0x63c85b8, 0xc04404de30}, 0x28, 0x28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a12def0, {0x63c85b8, 0xc04404de30}, {0xc06b9a1c20, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a12def0, {0x63c85b8, 0xc04404de30}, 0xc02f9fe320, 0xc09613b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04404de30}, 0x660089, {0xc06b9a1c20, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a284c00, {0x63c85b8, 0xc04404de30}, 0xc0388ff900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04404dda0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04404dda0}, {0xc09613d3f8, 0x660089}, 0xc09613d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04404dda0}, {0xc083b85140}, 0xc0388ff600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04404dd10}, 0xc0388ff600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}, 0xc075103300, 0xc01ec016b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}, 0x3, 0xc06596d8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc09613d980}}}, {0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}, 0xc075103300, 0xc06596d8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04404dd10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04404dd10}, {0x40, 0xc09613db70}, 0xc09613dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}, 0x203019, 0x203019) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04404dd10}, {0x4e9d7a0, 0xc0388ff600}, 0xc075103300, 0xc01ec016b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04404dd10}, 0xc0c2845140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af938c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af938c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795065 [runnable]: sync.runtime_SemacquireMutex(0xc07c21c690, 0x8f, 0x7fdb0c711b40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc07c21c538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc07c21c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc05f7be480, {0xffffffffffffff15, 0xc07c21c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc05f7be480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc05f7be480, {0xc02cf6e600, 0x32, 0x80}, {0xc02e7d7e00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc086481998, {0xc02cf6e600, 0x32, 0x80}, {0xc02e7d7e00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0864802c0, {0xc02e7d7e00, 0x72, 0x80}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0864802c0, {0xc02cf6e600, 0x72, 0xc02e7d7e00}, {0xc02e7d7e00, 0xc0663a9220, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc086480000, {0xc02e7d7e00, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04d024ea8, {{0xc0663a9220, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc023a52800, {{0xc04c20d4a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc06aca7400, {0x63c85b8, 0xc047bdc420}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc047bdc420}, {0x64a6560, 0xc023a52800}, {0xc04c20d4a0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc047bdc420}, {0x6466ed8, 0xc04d024c00}, {0xc04c20d4a0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04612cfd8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc056f6c700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02008fc00, {0x63c85b8, 0xc047bdc420}, 0xc056f6c700, 0xc02c9ff900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02008fc00, {0x63c85b8, 0xc047bdc420}, 0xc056f6c700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047bdc390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047bdc390}, {0xc0110d48f0, 0x660089}, 0xc0110d48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047bdc390}, {0xc00102e4e0}, 0xc056f6c500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047bdc330}, 0xc056f6c500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc022555650}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809846 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc022bfc360}, 0xc02f3ddd28, 0x1, 0x70801805d7ba68, 0x37e11d600, 0xc073f77020, 0xc082a3a3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc022bfc360}, 0x120, 0xc027bb5230, 0x73f76fc0, 0x0, 0x660069, 0xc073f77020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc022bfc360}, 0xc073f76fc0, {{{{0x0, 0x0}, {0xc066688480, 0x18}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc022bfc360}, 0x6352410, 0x4b1bd210) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc022bfc360}, {0xc073f76ea0, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc022bfc360}, 0xc05ac76fa0, 0xc027bb59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022bfc360}, 0x660089, {0xc073f76ea0, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc022bfc360}, 0xc02927d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022bfc1b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022bfc1b0}, {0xc027bb73f8, 0x660089}, 0xc027bb7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022bfc1b0}, {0x0}, 0xc02927d100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022bfc0c0}, 0xc02927d100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}, 0xc03bcc6340, 0xc02f3ddcf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}, 0x3, 0xc03f66c040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc027bb7980}}}, {0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}, 0xc03bcc6340, 0xc03f66c040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022bfc0c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022bfc0c0}, {0x40, 0xc027bb7b70}, 0xc027bb7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022bfc0c0}, {0x4e9d7a0, 0xc02927d100}, 0xc03bcc6340, 0xc02f3ddcf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022bfc0c0}, 0xc0a3017b60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc073f76d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc073f76d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219687872 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff36660, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc08c24b480, 0xc082daf000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc08c24b480, {0xc082daf000, 0x2b7e, 0x2b7e}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc08c24b480, {0xc082daf000, 0x2b7e, 0xc05168f2c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01a06e270, {0xc082daf000, 0x7eda79, 0xc082daf005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc017ca7260, {0xc082daf000, 0xc07dac6c60, 0x2b7e}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02febe888, {0xc082daf000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc072a673f8, {0x631b900, 0xc02febe888}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc072a67180, {0x7fe42ab59310, 0xc017ca7260}, 0x2b7e) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc072a67180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc072a67180, {0xc090bae000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06fb9cb40, {0xc090bae000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc084e02088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc084e02088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc084e02660, {0x634aac0, 0xc084e02088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc084e02000, 0xc01fe94b90, 0xc0856b342b, 0xc0856b342d, {0x635bec0, 0xc017971830}, 0xc000d6ae00, 0xc0399c1ee8, 0xc0856b3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc084e02000, {0x63c85b8, 0xc0632dd6b0}, 0xc01fe94b80, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a582a046, 0x8}}, 0x0, 0xc0632dc900, 0xc0632dd290, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0632dd6b0}, {0x643d810, 0xc017ca7260}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc041f5a6c0}, {0x643d810, 0xc017ca7260}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc041f5a6c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804988 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc03897be00}, 0xc08afc7590, 0x1, 0x0, 0x1, 0xc055b21500, 0xc057ce2600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03897be00}, 0x1e0, 0xc094ada728, 0x55b214a0, 0xc018c8a000, 0x18, 0xc055b21500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc03897be00}, 0xc055b214a0, {{{{0x0, 0x0}, {0xc056cb4fc0, 0x3a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03897be00}, 0x0, 0x64c369b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc03897be00}, {0xc07422d200, {0x174301103dc81678, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc03897be00}, 0xc07c3395e0, 0xc064c36ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03897be00}, 0x660089, {0xc07422d200, {0x174301103dc81678, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc03897be00}, 0xc06f10be00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103dc81678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103dc81678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03897bd40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03897bd40}, {0xc002fe68f0, 0x660089}, 0xc002fe68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03897bd40}, {0xc00102e4e0}, 0xc06f10bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03897bbf0}, 0xc06f10bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02a0b7860}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801340 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc023d7c800, {0x63c85b8, 0xc036f3a360}, 0xc06549ef00, 0xc07ad91180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023d7c800, {0x63c85b8, 0xc036f3a360}, 0xc06549ef00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101adfa430, 0x0, 0x0}, 0xc04afb9350, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101adfa430, 0x0, 0x0}, 0xc04afb9350, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc036f3a2d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc036f3a2d0}, {0xc052a333f8, 0x660089}, 0xc052a33400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc036f3a2d0}, {0x27}, 0xc06549ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc036f3a240}, 0xc06549ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}, 0xc049200040, 0xc01d7e37e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}, 0x3, 0xc0c63d36c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc052a33980}}}, {0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}, 0xc049200040, 0xc0c63d36c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc036f3a240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc036f3a240}, {0x40, 0xc052a33b70}, 0xc052a33ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc036f3a240}, {0x4e9d7a0, 0xc06549ed00}, 0xc049200040, 0xc01d7e37e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc036f3a240}, 0xc0c8003800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ab857440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ab857440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806433 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021d53400, {0x63c85b8, 0xc08c4125a0}, 0xc05b131200, 0xc041b81400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021d53400, {0x63c85b8, 0xc08c4125a0}, 0xc05b131200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104f8effd0, 0x0, 0x0}, 0xc08a6381e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104f8effd0, 0x0, 0x0}, 0xc08a6381e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08c412510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08c412510}, {0xc070135a08, 0x660089}, 0xc070135a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08c412510}, {0xc00102e4e0}, 0xc05b131100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08c4124b0}, 0xc05b131100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc08d3c0c00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc08d3c0c00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809719 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc012025000, {0x63c85b8, 0xc044c04b10}, 0xc0646b5a00, 0xc024ef8dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc012025000, {0x63c85b8, 0xc044c04b10}, 0xc0646b5a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106815af69, 0x0, 0x0}, 0xc066f2e320, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106815af69, 0x0, 0x0}, 0xc066f2e320, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044c049c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044c049c0}, {0xc00df473f8, 0x660089}, 0xc00df47400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044c049c0}, {0x0}, 0xc0646b5700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044c04900}, 0xc0646b5700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}, 0xc01e142fa0, 0xc049760ca8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}, 0x3, 0xc04ad14b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00df47980}}}, {0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}, 0xc01e142fa0, 0xc04ad14b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044c04900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044c04900}, {0x40, 0xc00df47b70}, 0xc00df47ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044c04900}, {0x4e9d7a0, 0xc0646b5700}, 0xc01e142fa0, 0xc049760ca8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044c04900}, 0xc01cefe3c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc057c23c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc057c23c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809945 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc023afbc00, {0x63c85b8, 0xc042fc7770}, 0xc0b5d8ab00, 0xc068976b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023afbc00, {0x63c85b8, 0xc042fc7770}, 0xc0b5d8ab00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106ce9233c, 0x0, 0x0}, 0xc044115600, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106ce9233c, 0x0, 0x0}, 0xc044115600, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042fc75c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042fc75c0}, {0xc02807b3f8, 0x660089}, 0xc02807b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042fc75c0}, {0x3feffe1328e8ab1a}, 0xc0b5d8a800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042fc7530}, 0xc0b5d8a800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}, 0xc0267a6440, 0xc07289a840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}, 0x3, 0xc01b5f4340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02807b980}}}, {0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}, 0xc0267a6440, 0xc01b5f4340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc042fc7530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042fc7530}, {0x40, 0xc02807bb70}, 0xc02807bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc042fc7530}, {0x4e9d7a0, 0xc0b5d8a800}, 0xc0267a6440, 0xc07289a840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc042fc7530}, 0xc0983d04e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04f75c6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04f75c6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 202012963 [select, 31 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219812705 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc0367780f0}, 0xc0c05c37e8, 0x1, 0xc03cebb0a8, 0x519bbc, 0xc052733e00, 0xc08792f3e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05afa9480, {0x63c85b8, 0xc0367780f0}, 0xc081cef0e0, 0xc03cebb1a8, 0x52733e00, 0x0, 0x0, 0xc052733e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc0367780f0}, 0xc052733d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc059239680, 0x4e}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0367780f0}, 0x6352260, 0x47c525b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc0367780f0}, {0xc052733c20, {0x17430110617803ba, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc0367780f0}, 0xc072e4c000, 0xc03cebb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0367780f0}, 0x660089, {0xc052733c20, {0x17430110617803ba, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0367780f0}, 0xc08256be00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110617803ba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110617803ba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc036778060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc036778060}, {0xc03cebd3f8, 0x660089}, 0xc03cebd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc036778060}, {0xc03af40c30}, 0xc08256bc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070dbbec0}, 0xc08256bc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}, 0xc0b453db40, 0xc04b12a108) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}, 0x3, 0xc08e517400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03cebd980}}}, {0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}, 0xc0b453db40, 0xc08e517400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070dbbec0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070dbbec0}, {0x40, 0xc03cebdb70}, 0xc03cebdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070dbbec0}, {0x4e9d7a0, 0xc08256bc00}, 0xc0b453db40, 0xc04b12a108) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070dbbec0}, 0xc056d3b080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc052733b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc052733b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806615 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc08a8c8960}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810340 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc023ba7c00, {0x63c85b8, 0xc027a02150}, 0xc01ca69900, 0xc07a98a640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023ba7c00, {0x63c85b8, 0xc027a02150}, 0xc01ca69900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c8ec196, 0x0, 0x0}, 0xc04a75f710, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c8ec196, 0x0, 0x0}, 0xc04a75f710, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027a020c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027a020c0}, {0xc010d6f3f8, 0x660089}, 0xc010d6f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027a020c0}, {0xc01b5bc820}, 0xc01ca69700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027a02000}, 0xc01ca69700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}, 0xc07a403aa0, 0xc04a2d4ab0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}, 0x3, 0xc0bdce2700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010d6f980}}}, {0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}, 0xc07a403aa0, 0xc0bdce2700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc027a02000}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027a02000}, {0x40, 0xc010d6fb70}, 0xc010d6fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc027a02000}, {0x4e9d7a0, 0xc01ca69700}, 0xc07a403aa0, 0xc04a2d4ab0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc027a02000}, 0xc0b66bb9e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0223606c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0223606c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809625 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc041630540}, 0xc0292d4720, 0x1, 0x0, 0x1, 0xc08274a060, 0xc0744d8840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041630540}, 0x120, 0xc027f32728, 0x8274a000, 0x0, 0x660069, 0xc08274a060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc041630540}, 0xc08274a000, {{{{0x0, 0x0}, {0xc09ed7eb40, 0xf1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041630540}, 0x6352410, 0x43391978) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc041630540}, {0xc07dfdf7a0, {0x1743011067de4198, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc041630540}, 0xc0495779a0, 0xc07c63eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041630540}, 0x660089, {0xc07dfdf7a0, {0x1743011067de4198, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc041630540}, 0xc05feb6000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067de4198, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067de4198, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0416304b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0416304b0}, {0xc0abc4a8f0, 0x660089}, 0xc0abc4a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0416304b0}, {0xc00102e4e0}, 0xc0bd5b9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041630450}, 0xc0bd5b9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0352e5080}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801777 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x6d) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x60, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc03f3125f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc033bfae00, {0xc01c3d6b80, 0x32, 0x77}, {0xc023d15c00, 0x32, 0x74}, 0x1, 0x25) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc03f312488, {0xc01c3d6b80, 0x1, 0x74}, {0xc023d15c00, 0x2d8, 0xc096e54790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc03f312488, {0xc01c3d6b80, 0x32, 0x77}, {0xc023d15c00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc03f311bc0, {0xc023d15c00, 0x74, 0x77}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc03f311bc0, {0xc01c3d6b80, 0x74, 0xc023d15c00}, {0xc023d15c00, 0xc0504c3ea0, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc03f311900, {0xc023d15c00, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0564e6828, {{0xc0504c3ea0, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc052467e00, {{0xc0b9011e30, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc03fa58400, {0x63c85b8, 0xc041cea810}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc041cea810}, {0x64a6560, 0xc052467e00}, {0xc0b9011e30, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc041cea810}, {0x6466ed8, 0xc0564e6580}, {0xc0b9011e30, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc054221fb0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01eb22200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020eb8000, {0x63c85b8, 0xc041cea810}, 0xc01eb22200, 0xc02d48dae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020eb8000, {0x63c85b8, 0xc041cea810}, 0xc01eb22200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041cea780}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041cea780}, {0xc0076c68f0, 0x660089}, 0xc0076c68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041cea780}, {0xc00102e4e0}, 0xc01eb22100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041cea720}, 0xc01eb22100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618cb110}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809943 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030065c00, {0x63c85b8, 0xc071c0dad0}, 0xc078875900, 0xc07c610a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc071c0dad0}, 0xc078875900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bfdea12, 0x0, 0x0}, 0xc044114c80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bfdea12, 0x0, 0x0}, 0xc044114c80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc071c0da40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071c0da40}, {0xc00df473f8, 0x660089}, 0xc00df47400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc071c0da40}, {0x7fe429311c20}, 0xc078875500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc071c0d9b0}, 0xc078875500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}, 0xc065635040, 0xc0269f45a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}, 0x3, 0xc078e88f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00df47980}}}, {0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}, 0xc065635040, 0xc078e88f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc071c0d9b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071c0d9b0}, {0x40, 0xc00df47b70}, 0xc00df47ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc071c0d9b0}, {0x4e9d7a0, 0xc078875500}, 0xc065635040, 0xc0269f45a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc071c0d9b0}, 0xc0524046c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059024900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059024900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806555 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4d4a8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07d43be00, 0xc04d0ad000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07d43be00, {0xc04d0ad000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07d43be00, {0xc04d0ad000, 0x735, 0xc06eb04640}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000f06958, {0xc04d0ad000, 0x7eda79, 0xc04d0ad005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0adbb78c0, {0xc04d0ad000, 0x432c000000000000, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04a6d9758, {0xc04d0ad000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0bd09ecf8, {0x631b900, 0xc04a6d9758}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0bd09ea80, {0x7fe42ab59310, 0xc0adbb78c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0bd09ea80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0bd09ea80, {0xc09a8ef000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06519a600, {0xc09a8ef000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc063792088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc063792088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc063792660, {0x634aac0, 0xc063792088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc063792000, 0xc047727030, 0xc06cabf42b, 0xc06cabf42d, {0x635bec0, 0xc08ff8fef0}, 0xc000d6ae00, 0xc0896cb114, 0xc06cabf429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc063792000, {0x63c85b8, 0xc08c6ca3c0}, 0xc047726f90, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc096275009, 0x8}}, 0x0, 0xc08c6ca360, 0xc08c6ca390, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc08c6ca3c0}, {0x643d810, 0xc0adbb78c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc08bb70cf0}, {0x643d810, 0xc0adbb78c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc08bb70cf0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805366 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016d6d2c0, {0x63c85b8, 0xc07adfc990}, 0xc0395cf9b0, 0x1, 0xc086eed0a8, 0x519bbc, 0xc05a86f140, 0xc059e65260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07adfc990}, 0x120, 0xc086eed230, 0x5a86f0e0, 0xc018c8bd40, 0x18, 0xc05a86f140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016d6d2c0, {0x63c85b8, 0xc07adfc990}, 0xc05a86f0e0, {{{{0x0, 0x0}, {0xc00ab9a000, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07adfc990}, 0x6352410, 0x52e78500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce3180, {0x63c85b8, 0xc07adfc990}, {0xc05a86efc0, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce3180, {0x63c85b8, 0xc07adfc990}, 0xc0205a2280, 0xc086eed9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07adfc990}, 0x660089, {0xc05a86efc0, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016efac00, {0x63c85b8, 0xc07adfc990}, 0xc05074ad00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07adfc900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07adfc900}, {0xc086eef3f8, 0x660089}, 0xc086eef400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07adfc900}, {0xc0015221e0}, 0xc05074ab00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07adfc870}, 0xc05074ab00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}, 0xc061d283e0, 0xc0274bc990) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}, 0x3, 0xc06d5bcd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc086eef980}}}, {0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}, 0xc061d283e0, 0xc06d5bcd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07adfc870}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07adfc870}, {0x40, 0xc086eefb70}, 0xc086eefab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07adfc870}, {0x4e9d7a0, 0xc05074ab00}, 0xc061d283e0, 0xc0274bc990) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07adfc870}, 0xc018abd5c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a49d200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a49d200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799380 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa0b48, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06e2bf080, 0xc0234d2800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06e2bf080, {0xc0234d2800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06e2bf080, {0xc0234d2800, 0x739, 0xc028ce3e00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0351a8618, {0xc0234d2800, 0x7eda79, 0xc0234d2805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04de125a0, {0xc0234d2800, 0xc084e5ac60, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc026a0ab88, {0xc0234d2800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0478b7e78, {0x631b900, 0xc026a0ab88}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0478b7c00, {0x7fe42ab59310, 0xc04de125a0}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0478b7c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0478b7c00, {0xc09d8da000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc05b8a40f0, {0xc09d8da000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc091c6eb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc091c6eb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc091c6f0e0, {0x634aac0, 0xc091c6eb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc091c6ea80, 0xc047053180, 0xc084e5b42b, 0xc084e5b42d, {0x635bec0, 0xc00107e750}, 0xc000d6ae00, 0xc0580989e8, 0xc084e5b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc091c6ea80, {0x63c85b8, 0xc060c16b10}, 0xc047053170, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a7ac9009, 0xb}}, 0x0, 0xc060c16ab0, 0xc060c16ae0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc060c16b10}, {0x643d810, 0xc04de125a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02be99bf0}, {0x643d810, 0xc04de125a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02be99bf0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808421 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01f61dc00, {0x63c85b8, 0xc018160e10}, 0xc0c9185200, 0xc0627cc0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f61dc00, {0x63c85b8, 0xc018160e10}, 0xc0c9185200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102097aece, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102097aece, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc018160cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018160cf0}, {0xc07ca6da28, 0x660089}, 0xc07ca6da30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc018160cf0}, {0xc00102e4e0}, 0xc0c9185100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018160ba0}, 0xc0c9185100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc042408dd0, {0x63c85b8, 0xc0181608d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc042408dd0, {0x63c85b8, 0xc0181608d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0181608d0}, {0x4f67ad2, 0x6446ff8}, 0xc016471fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219802956 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0180cc7b0}, 0xc044c13c68, 0x1, 0xc05350b0a8, 0x519bbc, 0xc040913c80, 0xc03c42de60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0180cc7b0}, 0x1e0, 0xc05350b230, 0x40913c20, 0x0, 0x660069, 0xc040913c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0180cc7b0}, 0xc040913c20, {{{{0x0, 0x0}, {0xc04f74ad80, 0xd3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0180cc7b0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0180cc7b0}, {0xc019c859e0, {0x17430110209da4f8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0180cc7b0}, 0xc05b6343c0, 0xc05350b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0180cc7b0}, 0x660089, {0xc019c859e0, {0x17430110209da4f8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0180cc7b0}, 0xc0b91ccf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110209da4f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110209da4f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0180cc720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0180cc720}, {0xc05350d3f8, 0x660089}, 0xc05350d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0180cc720}, {0xc07489f410}, 0xc0b91ccd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0180cc690}, 0xc0b91ccd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}, 0xc04678ac40, 0xc047b7cae0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}, 0x3, 0xc01f67b4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05350d980}}}, {0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}, 0xc04678ac40, 0xc01f67b4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0180cc690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0180cc690}, {0x40, 0xc05350db70}, 0xc05350dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0180cc690}, {0x4e9d7a0, 0xc0b91ccd00}, 0xc04678ac40, 0xc047b7cae0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0180cc690}, 0xc027ff26c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc019c858c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc019c858c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812584 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803681 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035680, {0x63c85b8, 0xc04ff92960}, 0xc0292493e0, 0x1, 0x0, 0x1, 0xc0433f03c0, 0xc0429463c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04ff92960}, 0x120, 0xc076c66728, 0x433f0360, 0x0, 0x660069, 0xc0433f03c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035680, {0x63c85b8, 0xc04ff92960}, 0xc0433f0360, {{{{0x0, 0x0}, {0xc0bac66360, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ff92960}, 0x6352410, 0x3080ff78) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd6d0, {0x63c85b8, 0xc04ff92960}, {0xc042f3ec60, {0x174301102e67b889, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd6d0, {0x63c85b8, 0xc04ff92960}, 0xc0779f70e0, 0xc08da96ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ff92960}, 0x660089, {0xc042f3ec60, {0x174301102e67b889, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc04ff92960}, 0xc04b2e3f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e67b889, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e67b889, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ff928d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ff928d0}, {0xc07d2568f0, 0x660089}, 0xc07d2568f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ff928d0}, {0xc00102e4e0}, 0xc04b2e3d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ff92840}, 0xc04b2e3d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc079925b90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219792871 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a59e50, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04f321f80, 0xc01f00a800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04f321f80, {0xc01f00a800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04f321f80, {0xc01f00a800, 0x735, 0xc0387c3cc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03824c3e8, {0xc01f00a800, 0x7eda79, 0xc01f00a805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0753abe60, {0xc01f00a800, 0x46a86f, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a903f0, {0xc01f00a800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc09d0b5778, {0x631b900, 0xc043a903f0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc09d0b5500, {0x7fe42ab59310, 0xc0753abe60}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc09d0b5500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc09d0b5500, {0xc084b16000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0265f4198, {0xc084b16000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc059046b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc059046b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0590470e0, {0x634aac0, 0xc059046b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc059046a80, 0xc03fc92090, 0xc08586f42b, 0xc08586f42d, {0x635bec0, 0xc033446480}, 0xc000d6ae00, 0xc048e3a488, 0xc08586f429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc059046a80, {0x63c85b8, 0xc044e8c7e0}, 0xc03fc92080, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc021b2b036, 0x8}}, 0x0, 0xc044e8c780, 0xc044e8c7b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc044e8c7e0}, {0x643d810, 0xc0753abe60}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01c6179b0}, {0x643d810, 0xc0753abe60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01c6179b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805771 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc065f68ab0}, 0xc08bb52b28, 0x1, 0xc006c690a8, 0x519bbc, 0xc0384a60c0, 0xc056a03bc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc058c519e0, {0x63c85b8, 0xc065f68ab0}, 0xc058c51a10, 0xc006c691a8, 0x41, 0x60, 0xffffffffffffffff, 0xc0384a60c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc065f68ab0}, 0xc0384a6000, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc073d51560, 0x19}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc065f68ab0}, 0x6352260, 0x4040cb50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc065f68ab0}, {0xc04607e120, {0x1743010ff25f28bf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc065f68ab0}, 0xc0442c95e0, 0xc006c699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc065f68ab0}, 0x660089, {0xc04607e120, {0x1743010ff25f28bf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc065f68ab0}, 0xc053e13100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff25f28bf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff25f28bf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc065f68a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc065f68a20}, {0xc006c6b3f8, 0x660089}, 0xc006c6b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc065f68a20}, {0x0}, 0xc053e12f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc065f68900}, 0xc053e12f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}, 0xc029f23180, 0xc08bb52cc0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}, 0x3, 0xc0068c6a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006c6b980}}}, {0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}, 0xc029f23180, 0xc0068c6a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc065f68900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc065f68900}, {0x40, 0xc006c6bb70}, 0xc006c6bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc065f68900}, {0x4e9d7a0, 0xc053e12f00}, 0xc029f23180, 0xc08bb52cc0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc065f68900}, 0xc05efd0cc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04d5e6120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04d5e6120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804443 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc03fe9fc20}, 0xc04f13e5a0, 0x1, 0xc06572b0a8, 0x519bbc, 0xc04e361140, 0xc04e076ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03fe9fc20}, 0x1e0, 0xc06572b230, 0x4e3610e0, 0x0, 0x660069, 0xc04e361140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc03fe9fc20}, 0xc04e3610e0, {{{{0x0, 0x0}, {0xc01fe83440, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03fe9fc20}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc03fe9fc20}, {0xc05994cd80, {0x1743011032cd4a27, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc03fe9fc20}, 0xc057552c80, 0xc06572b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03fe9fc20}, 0x660089, {0xc05994cd80, {0x1743011032cd4a27, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc03fe9fc20}, 0xc04eeb8000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032cd4a27, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032cd4a27, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03fe9fb30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fe9fb30}, {0xc06572d3f8, 0x660089}, 0xc06572d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03fe9fb30}, {0x4ed3465}, 0xc054e6be00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03fe9faa0}, 0xc054e6be00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}, 0xc055902ce0, 0xc04dbe5dd0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}, 0x3, 0xc0c3354f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06572d980}}}, {0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}, 0xc055902ce0, 0xc0c3354f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03fe9faa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fe9faa0}, {0x40, 0xc06572db70}, 0xc06572dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03fe9faa0}, {0x4e9d7a0, 0xc054e6be00}, 0xc055902ce0, 0xc04dbe5dd0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03fe9faa0}, 0xc03e903f20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05994ca20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05994ca20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801392 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04e55b1a0}, 0xc024451908, 0x1, 0xc04d3a3ff0, 0x519bbc, 0xc0352e1140, 0xc03411a060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04e55b1a0}, 0x1e0, 0xc04d3a4178, 0x352e10e0, 0x0, 0x660069, 0xc0352e1140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04e55b1a0}, 0xc0352e10e0, {{{{0x0, 0x0}, {0xc05bd28fc0, 0xe9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04e55b1a0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04e55b1a0}, {0xc03fea2a20, {0x1743011019a7e848, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04e55b1a0}, 0xc052f77b80, 0xc08bf48938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e55b1a0}, 0x660089, {0xc03fea2a20, {0x1743011019a7e848, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04e55b1a0}, 0xc050942300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019a7e848, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019a7e848, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e55b110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e55b110}, {0xc08bf4a340, 0x660089}, 0xc08bf4a348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e55b110}, {0xc00102e4e0}, 0xc050942200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e55b0b0}, 0xc050942200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc04e55ab40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812767 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812297 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03005fa40, {0x63c85b8, 0xc066630db0}, 0xc027ceda58, 0x1, 0xc0387bd0a8, 0x519bbc, 0xc0778fe840, 0xc085b9f020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc066630db0}, 0x120, 0xc0387bd230, 0x778fe7e0, 0x0, 0x660069, 0xc0778fe840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03005fa40, {0x63c85b8, 0xc066630db0}, 0xc0778fe7e0, {{{{0x0, 0x0}, {0xc066c17e60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc066630db0}, 0x6352410, 0x36ce80a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0300079a0, {0x63c85b8, 0xc066630db0}, {0xc0778fe6c0, {0x174301107b6e3d85, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0300079a0, {0x63c85b8, 0xc066630db0}, 0xc04544c5a0, 0xc0387bd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066630db0}, 0x660089, {0xc0778fe6c0, {0x174301107b6e3d85, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0300a9000, {0x63c85b8, 0xc066630db0}, 0xc04fb2c100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b6e3d85, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b6e3d85, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066630d20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066630d20}, {0xc0387bf3f8, 0x660089}, 0xc0387bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066630d20}, {0x7fe38e2e76c0}, 0xc04f4b9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066630c90}, 0xc04f4b9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}, 0xc0405013e0, 0xc0254ed488) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}, 0x3, 0xc06aff2940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0387bf980}}}, {0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}, 0xc0405013e0, 0xc06aff2940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066630c90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066630c90}, {0x40, 0xc0387bfb70}, 0xc0387bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066630c90}, {0x4e9d7a0, 0xc04f4b9f00}, 0xc0405013e0, 0xc0254ed488) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066630c90}, 0xc016ce95c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc09019efc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc09019efc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791185 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc036745e00}, 0xc028b37cb0, {0x646cc48, 0xc036745ec0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc036745e00}, 0xedb7a834e, 0xc0629db320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc036745e00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc049628c60}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc028b37ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc049628c60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812576 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219347380 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc05c7fcc60, {0x63c85b8, 0xc0501bb680}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc06e21e600, {0x63c85b8, 0xc0501bb680}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc06e21e600, {0x63c8510, 0xc024980040}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc06e21e600, {0x63c8510, 0xc024980040}, 0xc03f9fc500, {0x7fe42b790cf0, 0xc040203bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc06e21e600, {0x63c85b8, 0xc03a358ab0}, {{0x63f88f8, 0xc03f9fc500}, {0xc035705d92, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc06e21e600, {0x63c85b8, 0xc03a358ab0}, {{0x63f88f8, 0xc03f9fc500}, {0xc035705d92, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc03f9fc500}, {0xc035705d92, 0x23}, 0x0, 0x0}, {0x1557eb97, 0xedb7a8350, 0x0}, {0x1557f5d5, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06e21e600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06e21e600, {0x63c8510, 0xc0d5489880}, 0xc06c885dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a3d3a80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0d5489880}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a3d3a80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801780 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc06fbd0ed0}, 0xc04bf53bc0, 0x1, 0xc058ca70a8, 0x519bbc, 0xc03411bc80, 0xc033323500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06fbd0ed0}, 0x1e0, 0xc058ca7230, 0x3411bc20, 0x0, 0x660069, 0xc03411bc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc06fbd0ed0}, 0xc03411bc20, {{{{0x0, 0x0}, {0xc04f93eb40, 0x46}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06fbd0ed0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc06fbd0ed0}, {0xc014f80000, {0x174301101d2ea197, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc06fbd0ed0}, 0xc0b202a1e0, 0xc058ca79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06fbd0ed0}, 0x660089, {0xc014f80000, {0x174301101d2ea197, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc06fbd0ed0}, 0xc04b6ca700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d2ea197, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d2ea197, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06fbd0e40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fbd0e40}, {0xc058ca93f8, 0x660089}, 0xc058ca9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06fbd0e40}, {0x7fdd4a34b888}, 0xc04b6ca500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fbd0db0}, 0xc04b6ca500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}, 0xc07ab7fb80, 0xc09d75ced0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}, 0x3, 0xc0ce8ef300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc058ca9980}}}, {0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}, 0xc07ab7fb80, 0xc0ce8ef300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06fbd0db0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fbd0db0}, {0x40, 0xc058ca9b70}, 0xc058ca9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06fbd0db0}, {0x4e9d7a0, 0xc04b6ca500}, 0xc07ab7fb80, 0xc09d75ced0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06fbd0db0}, 0xc042d27ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04a564120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04a564120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800576 [runnable]: sync.runtime_SemacquireMutex(0xc09445c690, 0x8f, 0xc0254bdb80) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x191c615, 0x7fe3ec794380, 0x1194e0b) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x2a, 0x2a, 0x33, 0xc09445c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc045087200, {0xffffffffffffff15, 0xc09445c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc045087200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc045087200, {0xc00e04e9a0, 0x32, 0x6d}, {0xc0312a9a00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0a4b07298, {0xc00e04e9a0, 0x32, 0x6d}, {0xc0312a9a00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0a4b05bc0, {0xc0312a9a00, 0x74, 0x6d}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0a4b05bc0, {0xc00e04e9a0, 0x74, 0xc0312a9a00}, {0xc0312a9a00, 0xc048f8f9a0, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0a4b05900, {0xc0312a9a00, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc052cecea8, {{0xc048f8f9a0, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc03b047b00, {{0xc0de1702d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc06ac89800, {0x63c85b8, 0xc0759d79b0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0759d79b0}, {0x64a6560, 0xc03b047b00}, {0xc0de1702d0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0759d79b0}, {0x6466ed8, 0xc052cecc00}, {0xc0de1702d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0776d8078}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0648cd800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b11c00, {0x63c85b8, 0xc0759d79b0}, 0xc0648cd800, 0xc0977be1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b11c00, {0x63c85b8, 0xc0759d79b0}, 0xc0648cd800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0759d7920}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0759d7920}, {0xc0562c48f0, 0x660089}, 0xc0562c48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0759d7920}, {0xc00102e4e0}, 0xc0648cd700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0759d78c0}, 0xc0648cd700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc021185f80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808713 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0310bf000, {0x63c85b8, 0xc028f2cf90}, 0xc043833100, 0xc053b57180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310bf000, {0x63c85b8, 0xc028f2cf90}, 0xc043833100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011055697687, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011055697687, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc028f2ced0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028f2ced0}, {0xc00da353f8, 0x660089}, 0xc00da35400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc028f2ced0}, {0x29}, 0xc043832f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc028f2ce40}, 0xc043832f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}, 0xc056b74320, 0xc053c6fc80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}, 0x3, 0xc041bcddc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00da35980}}}, {0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}, 0xc056b74320, 0xc041bcddc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc028f2ce40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028f2ce40}, {0x40, 0xc00da35b70}, 0xc00da35ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc028f2ce40}, {0x4e9d7a0, 0xc043832f00}, 0xc056b74320, 0xc053c6fc80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc028f2ce40}, 0xc02cbf7500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc067a5cfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc067a5cfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811534 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc079d7d3b0}, 0xc025186ea0, 0x1, 0xc03456d0a8, 0x519bbc, 0xc0522af380, 0xc012fa6a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc079d7d3b0}, 0x120, 0xc03456d230, 0x522af320, 0x0, 0x660069, 0xc0522af380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc079d7d3b0}, 0xc0522af320, {{{{0x0, 0x0}, {0xc09ff56fc0, 0x25}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc079d7d3b0}, 0x6352410, 0x33df4958) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc079d7d3b0}, {0xc0522af200, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc079d7d3b0}, 0xc06f9666e0, 0xc03456d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc079d7d3b0}, 0x660089, {0xc0522af200, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc079d7d3b0}, 0xc0c57bc800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079d7d320}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079d7d320}, {0xc03456f3f8, 0x660089}, 0xc03456f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079d7d320}, {0xc01a92a890}, 0xc0c57bc600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079d7d260}, 0xc0c57bc600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}, 0xc055083000, 0xc0c573dcc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}, 0x3, 0xc03f34bec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03456f980}}}, {0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}, 0xc055083000, 0xc03f34bec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079d7d260}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079d7d260}, {0x40, 0xc03456fb70}, 0xc03456fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079d7d260}, {0x4e9d7a0, 0xc0c57bc600}, 0xc055083000, 0xc0c573dcc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079d7d260}, 0xc03694f8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0522aeea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0522aeea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803292 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc060f43950}, 0xc07453f300, 0xc052cf17c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc060f43950}, 0xc07453f300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bd75419, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bd75419, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060f438c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060f438c0}, {0xc05725f3f8, 0x660089}, 0xc05725f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060f438c0}, {0xc0015221e0}, 0xc07453f000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc060f43800}, 0xc07453f000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}, 0xc06e7cc820, 0xc0592bb770) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}, 0x3, 0xc042e9b000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05725f980}}}, {0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}, 0xc06e7cc820, 0xc042e9b000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc060f43800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060f43800}, {0x40, 0xc05725fb70}, 0xc05725fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc060f43800}, {0x4e9d7a0, 0xc07453f000}, 0xc06e7cc820, 0xc0592bb770) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc060f43800}, 0xc0787b6ea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc035969d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc035969d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 214268260 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafeea4b0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05ef55f00, 0xc03a8bd000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05ef55f00, {0xc03a8bd000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05ef55f00, {0xc03a8bd000, 0x735, 0xc04dae08c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc029b7a0c8, {0xc03a8bd000, 0x7eda79, 0xc03a8bd005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02108bec0, {0xc03a8bd000, 0x3, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc076e945d0, {0xc03a8bd000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03b2c05f8, {0x631b900, 0xc076e945d0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03b2c0380, {0x7fe42ab59310, 0xc02108bec0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03b2c0380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03b2c0380, {0xc049b7d000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0532b7518, {0xc049b7d000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03d370b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03d370b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03d3710e0, {0x634aac0, 0xc03d370b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03d370a80, 0xc076fcdf00, 0xc04b57342b, 0xc04b57342d, {0x635bec0, 0xc03614c6c0}, 0xc000d6ae00, 0xc0b415b208, 0xc04b573429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03d370a80, {0x63c85b8, 0xc03f85fe60}, 0xc076fcdef0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc01fbb303b, 0x8}}, 0x0, 0xc03f85fe00, 0xc03f85fe30, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03f85fe60}, {0x643d810, 0xc02108bec0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01ac056b0}, {0x643d810, 0xc02108bec0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01ac056b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803293 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02efd4140, {0x63c85b8, 0xc06b092030}, 0xc030342150, 0x1, 0xc0761c50a8, 0x519bbc, 0xc036d15da0, 0xc0600b3380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06b092030}, 0x120, 0xc0761c5230, 0x36d15d40, 0x0, 0x660069, 0xc036d15da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02efd4140, {0x63c85b8, 0xc06b092030}, 0xc036d15d40, {{{{0x0, 0x0}, {0xc044e30240, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06b092030}, 0x6352410, 0x45bf4160) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ef76e60, {0x63c85b8, 0xc06b092030}, {0xc036d15c20, {0x17430110292b3b32, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ef76e60, {0x63c85b8, 0xc06b092030}, 0xc07076bf40, 0xc0761c59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b092030}, 0x660089, {0xc036d15c20, {0x17430110292b3b32, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02efcb000, {0x63c85b8, 0xc06b092030}, 0xc0afa50100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110292b3b32, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110292b3b32, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056135f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056135f80}, {0xc0761c73f8, 0x660089}, 0xc0761c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056135f80}, {0x7fe3b8b2fff8}, 0xc0517d5f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056135ef0}, 0xc0517d5f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}, 0xc0507583e0, 0xc07ab143c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}, 0x3, 0xc02b17d3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0761c7980}}}, {0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}, 0xc0507583e0, 0xc02b17d3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056135ef0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056135ef0}, {0x40, 0xc0761c7b70}, 0xc0761c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056135ef0}, {0x4e9d7a0, 0xc0517d5f00}, 0xc0507583e0, 0xc07ab143c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056135ef0}, 0xc0685437a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc036e49b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc036e49b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 214514445 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a5a300, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc01311f400, 0xc00de52800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc01311f400, {0xc00de52800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc01311f400, {0xc00de52800, 0x735, 0xc061cda280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc055acec28, {0xc00de52800, 0x7eda79, 0xc00de52805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc07b280ae0, {0xc00de52800, 0xc005416c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc05bebf6e0, {0xc00de52800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03f3e0cf8, {0x631b900, 0xc05bebf6e0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03f3e0a80, {0x7fe42ab59310, 0xc07b280ae0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03f3e0a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03f3e0a80, {0xc082b14000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0256c1c08, {0xc082b14000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc026351588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc026351588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc026351b60, {0x634aac0, 0xc026351588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc026351500, 0xc0134b8780, 0xc07c43d42b, 0xc07c43d42d, {0x635bec0, 0xc0435af9e0}, 0xc000d6ae00, 0xc05b63c478, 0xc07c43d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc026351500, {0x63c85b8, 0xc078fac840}, 0xc0134b8770, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc026dbf03c, 0x8}}, 0x0, 0xc078fac7e0, 0xc078fac810, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc078fac840}, {0x643d810, 0xc07b280ae0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06b798f60}, {0x643d810, 0xc07b280ae0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06b798f60}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812063 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01f506c00, {0x63c85b8, 0xc032ea73e0}, 0xc09dbff500, 0xc0b202be00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f506c00, {0x63c85b8, 0xc032ea73e0}, 0xc09dbff500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc032ea7350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc032ea7350}, {0xc0783ed858, 0x660089}, 0xc0783ed860) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc032ea7350}, {0xc00102e4e0}, 0xc09dbff400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc032ea72f0}, 0xc09dbff400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x174301107e0cfbb5, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc00f726a20}, 0x0, 0xc042643080) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc00f726a20}, 0x1c) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxnRecover(0xc00d502dc0, {0x63c85b8, 0xc00f726a20}, 0xc0518be7e0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:304 +0x135 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxn.func2({0x63c85b8, 0xc00f726a20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:160 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00f726a20}, {0x100000000000038, 0x20}, 0xc001b096e0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxn(0xc00d502dc0, 0xc0518be7e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:133 +0x133 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).ResolveIndeterminateCommit.func1() github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:96 +0x25 github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).doCall(0xc00d502e40, 0xc0722d5540, {0xc081ef5800, 0x24}, 0xc001b097b8) github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:128 +0x3b created by github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).DoChan github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:121 +0x2f1 goroutine 219812407 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fe65c00, {0x63c85b8, 0xc04f6fc930}, 0xc079621700, 0xc03544a500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fe65c00, {0x63c85b8, 0xc04f6fc930}, 0xc079621700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107ca83f7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107ca83f7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f6fc8a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f6fc8a0}, {0xc03ed333f8, 0x660089}, 0xc03ed33400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f6fc8a0}, {0x1700898901003d3d}, 0xc079621400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f6fc810}, 0xc079621400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}, 0xc0878866a0, 0xc043a91c98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}, 0x3, 0xc058616940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03ed33980}}}, {0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}, 0xc0878866a0, 0xc058616940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f6fc810}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f6fc810}, {0x40, 0xc03ed33b70}, 0xc03ed33ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f6fc810}, {0x4e9d7a0, 0xc079621400}, 0xc0878866a0, 0xc043a91c98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f6fc810}, 0xc04d5f7860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05396ca20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05396ca20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812550 [sleep]: time.Sleep(0x3b9aca00) GOROOT/src/runtime/time.go:193 +0x12e github.com/cockroachdb/cockroach/pkg/server.(*statusServer).CancelQueryByKey.func1() github.com/cockroachdb/cockroach/pkg/server/status.go:2871 +0x45 github.com/cockroachdb/cockroach/pkg/server.(*statusServer).CancelQueryByKey(0xc000a52c80, {0x63c85b8, 0xc0566143f0}, 0xc03f16a1f0) github.com/cockroachdb/cockroach/pkg/server/status.go:2882 +0x1aa github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_CancelQueryByKey_Handler.func1({0x63c85b8, 0xc0566143f0}, {0x4d07ba0, 0xc03f16a1f0}) github.com/cockroachdb/cockroach/pkg/server/serverpb/bazel-out/k8-opt/bin/pkg/server/serverpb/serverpb_go_proto_/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:8141 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}, 0xc0ccf7fb60, 0xc038161bd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:155 +0x643 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}, 0x3, 0xc071e548c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02790d980}}}, {0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}, 0xc0ccf7fb60, 0xc071e548c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056614360}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614360}, {0x40, 0xc02790db70}, 0xc02790dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056614360}, {0x4d07ba0, 0xc03f16a1f0}, 0xc0ccf7fb60, 0xc038161bd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_CancelQueryByKey_Handler({0x4eaf760, 0xc000a52c80}, {0x63c85b8, 0xc056614360}, 0xc071982300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/server/serverpb/bazel-out/k8-opt/bin/pkg/server/serverpb/serverpb_go_proto_/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:8143 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea27e0, 0xc00cbb1560, 0x91daa68, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea27e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803992 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc029a89110}, 0xc03b046d00, 0xc07401f0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc029a89110}, 0xc03b046d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032249dce, 0x0, 0x0}, 0xc045598f50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032249dce, 0x0, 0x0}, 0xc045598f50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029a89020}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029a89020}, {0xc0624453f8, 0x660089}, 0xc062445400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029a89020}, {0x0}, 0xc03b046b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029a88f90}, 0xc03b046b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}, 0xc03ef81840, 0xc041389860) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}, 0x3, 0xc0180d4f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062445980}}}, {0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}, 0xc03ef81840, 0xc0180d4f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029a88f90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029a88f90}, {0x40, 0xc062445b70}, 0xc062445ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029a88f90}, {0x4e9d7a0, 0xc03b046b00}, 0xc03ef81840, 0xc041389860) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029a88f90}, 0xc0425a3080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050d4f680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050d4f680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219685896 [IO wait]: internal/poll.runtime_pollWait(0x7fdb124bcd60, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc01202d600, 0xc00b56e000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc01202d600, {0xc00b56e000, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc01202d600, {0xc00b56e000, 0x739, 0xc0cd218780}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06af01a80, {0xc00b56e000, 0x7eda79, 0xc00b56e005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc047ed5f20, {0xc00b56e000, 0x0, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0342fcbd0, {0xc00b56e000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc08f5a2cf8, {0x631b900, 0xc0342fcbd0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc08f5a2a80, {0x7fe42ab59310, 0xc047ed5f20}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc08f5a2a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc08f5a2a80, {0xc04ad3b000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04bf8c528, {0xc04ad3b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc035b5d588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc035b5d588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc035b5db60, {0x634aac0, 0xc035b5d588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc035b5d500, 0xc04bbe7900, 0xc076ecb42b, 0xc076ecb42d, {0x635bec0, 0xc035837830}, 0xc000d6ae00, 0xc03d8a03d4, 0xc076ecb429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc035b5d500, {0x63c85b8, 0xc04a66f890}, 0xc04bbe78f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc044fb2053, 0xb}}, 0x0, 0xc04a66f830, 0xc04a66f860, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04a66f890}, {0x643d810, 0xc047ed5f20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc08a37c6c0}, {0x643d810, 0xc047ed5f20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc08a37c6c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219794607 [runnable]: sync.runtime_SemacquireMutex(0xc00c453198, 0x8f, 0x7fd5bcd22180) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc00c453040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc00c453068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc03cb1ed80, {0xffffffffffffff15, 0xc00c4531c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc03cb1ed80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc03cb1ed80, {0xc02c3e3d00, 0x32, 0x74}, {0xc02ae89880, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc079dee398, {0xc02c3e3d00, 0x32, 0x74}, {0xc02ae89880, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc079deccc0, {0xc02ae89880, 0x72, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc079deccc0, {0xc02c3e3d00, 0x72, 0xc02ae89880}, {0xc02ae89880, 0xc03b90b380, 0x31}, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc079deca00, {0xc02ae89880, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06daa42a8, {{0xc03b90b380, 0x31, 0x40}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0202bd600, {{0xc053f17230, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc060381800, {0x63c85b8, 0xc052284450}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc052284450}, {0x64a6560, 0xc0202bd600}, {0xc053f17230, 0x0, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc052284450}, {0x6466ed8, 0xc06daa4000}, {0xc053f17230, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01e069a70}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc056cc2000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020aef800, {0x63c85b8, 0xc052284450}, 0xc056cc2000, 0xc0d0452780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020aef800, {0x63c85b8, 0xc052284450}, 0xc056cc2000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0522843c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0522843c0}, {0xc00c4593f8, 0x660089}, 0xc00c459400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0522843c0}, {0x7fe34af92aa8}, 0xc067417c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc052284300}, 0xc067417c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}, 0xc0b906d360, 0xc041b3d650) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}, 0x3, 0xc0437a4280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00c459980}}}, {0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}, 0xc0b906d360, 0xc0437a4280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc052284300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052284300}, {0x40, 0xc00c459b70}, 0xc00c459ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc052284300}, {0x4e9d7a0, 0xc067417c00}, 0xc0b906d360, 0xc041b3d650) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc052284300}, 0xc03e2343c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04aceb200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04aceb200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812185 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fd49680, {0x63c85b8, 0xc031a1d2c0}, 0xc0204cc9d8, 0x1, 0x0, 0x1, 0xc05ba3af00, 0xc0441e1b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc031a1d2c0}, 0x120, 0xc032eb0728, 0x5ba3aea0, 0x0, 0x660069, 0xc05ba3af00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fd49680, {0x63c85b8, 0xc031a1d2c0}, 0xc05ba3aea0, {{{{0x0, 0x0}, {0xc08739b320, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc031a1d2c0}, 0x6352410, 0x54d3a2d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fc6b9a0, {0x63c85b8, 0xc031a1d2c0}, {0xc05ba3ad80, {0x1743011080a716be, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fc6b9a0, {0x63c85b8, 0xc031a1d2c0}, 0xc03c3ee960, 0xc07c238ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc031a1d2c0}, 0x660089, {0xc05ba3ad80, {0x1743011080a716be, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fd82400, {0x63c85b8, 0xc031a1d2c0}, 0xc06d230700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011080a716be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011080a716be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031a1ca80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031a1ca80}, {0xc0ad83c8f0, 0x660089}, 0xc0ad83c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031a1ca80}, {0xc00102e4e0}, 0xc06d230600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031a1c9f0}, 0xc06d230600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc031a1c6c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809255 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04ecac7b0}, 0xc01607bde8, 0x1, 0xc0210230a8, 0x519bbc, 0xc0347c0600, 0xc05564d500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04ecac7b0}, 0x1e0, 0xc021023230, 0x347c05a0, 0x0, 0x660069, 0xc0347c0600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04ecac7b0}, 0xc0347c05a0, {{{{0x0, 0x0}, {0xc086d65200, 0xd0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04ecac7b0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04ecac7b0}, {0xc051eb5d40, {0x1743011063a1674b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04ecac7b0}, 0xc065f2fc20, 0xc0210239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ecac7b0}, 0x660089, {0xc051eb5d40, {0x1743011063a1674b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04ecac7b0}, 0xc055d21e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063a1674b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063a1674b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ecac720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ecac720}, {0xc0210253f8, 0x660089}, 0xc021025400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ecac720}, {0x7fe323b7fbb8}, 0xc055d21c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ecac690}, 0xc055d21c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}, 0xc01c84a260, 0xc0139fcf78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}, 0x3, 0xc0b877b580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc021025980}}}, {0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}, 0xc01c84a260, 0xc0b877b580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ecac690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ecac690}, {0x40, 0xc021025b70}, 0xc021025ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ecac690}, {0x4e9d7a0, 0xc055d21c00}, 0xc01c84a260, 0xc0139fcf78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ecac690}, 0xc0553e42a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc051eb5c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc051eb5c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219776283 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc061235500}, {0xc0847e6908, 0x1, 0xc066fe29d0}, {0x1, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntent(0x0, {0x63c85b8, 0xc061235500}, {{{0xc0428dd890, 0x21, 0x30}, {0x0, 0x0, 0x0}}, {{0xd2, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:841 +0x9b github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc01d7f4f50, {0x63c85b8, 0xc061235500}, {0xc0735647e0, {0x174301106ba57473, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:549 +0x6db github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc061235500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc01d7f4f50, {0x63c85b8, 0xc061235500}, {0xc0735647e0, {0x174301106ba57473, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc061235500}, 0xc02e8d5b80, 0xc0847e7760) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc061235500}, 0xc02e8d5b80, {0xc0735647e0, {0x174301106ba57473, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc061235500}, 0xc062d7b200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106ba57473, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106ba57473, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc061235470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc061235470}, {0xc0770d1600, 0x660089}, 0xc0770d1608) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc061235470}, {0xc00102e4e0}, 0xc062d7b100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc061235410}, 0xc062d7b100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8510, 0xc02853ac40}, 0xe, 0xc06222cc00) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc07717b1e0, {0x63c8510, 0xc02853ac40}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/txn.go:696 +0x74 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc059ff2028, {0x63c8510, 0xc02853ac40}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:225 +0xb6 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc059ff2000, {{0x63c8510, 0xc02853ac40}, 0xc05214e700, 0xc0395179f0}) github.com/cockroachdb/cockroach/pkg/sql/update.go:196 +0x210 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc0bb3a9e78, {{0x63c8510, 0xc02853ac40}, 0xc05214e700, 0xc0395179f0}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc0770d5930, 0x63cb098}, {0x63caa08, 0xc0bb3a9e78}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:516 +0x147 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc0770d5930, {0x63caa08, 0xc0bb3a9e78}, {0x4ed688c, 0x5}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c9 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc0770d5930, {0x63caa08, 0xc0bb3a9e78}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x63c8510, 0xc02853ac40}, 0xc05214e700, 0xc0395179f0}, {0x63caa08, 0xc0bb3a9e78}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:519 +0x125 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc064769500, {0x63c8510, 0xc02853ac40}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:146 +0x87 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:120 +0x34 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x200000000) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc03efed440, {0x63c8510, 0xc02853ac40}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:119 +0x6b github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc03efed440, {0x63c8510, 0xc02853ac40}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:753 +0x3a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc043642680, {0x63c8510, 0xc02853ac40}, 0xc03efed440) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:469 +0x258 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc039517ce8, {0x63c8510, 0xc02853ac40}, 0xc07717b1e0) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:303 +0x205 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc02853a780}, 0xc044a4ac40, 0xc07717b1e0, 0xc068031780, 0xc0ae03c700, 0xc039517ce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc02853a780}, 0xc0395179f0, 0xc044a4ac40, 0xc06ca146c0, {{0x63caa08, 0xc0bb3a9e78}, 0x0}, 0xc0ae03c700) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc039517300, {0x63c8510, 0xc02853a780}, 0xc0395179f0, 0xc02853a780, {0x7fe42b790cf0, 0xc06ca146c0}, 0x0, 0xc0218d7738) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc039517300, {0x63c8510, 0xc02853a780}, 0xc0395179f0, {0x7fe42b790cf0, 0xc06ca146c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc039517300, {0x63c85b8, 0xc07412b080}, {{0x63f9d78, 0xc077d29600}, {0xc06e18db2b, 0x8b}, 0x3, 0x1}, 0xc06acdeea0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc039517300, {0x63c85b8, 0xc07412b080}, {{0x63f9d78, 0xc077d29600}, {0xc06e18db2b, 0x8b}, 0x3, 0x1}, 0xc06acdeea0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc039517300, {0x63c85b8, 0xc07412b080}, {0xc06acdeea0, {0xc0199c7590, 0x3, 0x3}, {0x9bf64bc, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc06e18dc34, 0x0}, 0x0, {0x14dd31f9, 0xedb7a8350, 0x0}, 0x1}, 0xc039517300, 0xc0847ef980, 0xc0847ef970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc039517300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc039517300, {0x63c8510, 0xc06576ec40}, 0xc08808fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc042ac1e50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06576ec40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc042ac1e50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219792243 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0b9a166c0}, 0xc08818fcb0, {0x646cc48, 0xc0b9a16780}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0b9a166c0}, 0xedb7a834e, 0xc06db32fc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0b9a166c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02f3d9d10}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc08818fef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc02f3d9d10}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219798862 [semacquire]: sync.runtime_SemacquireMutex(0x4d72379, 0x40, 0x7fe37f8602c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc000088000, 0xc0940b2410, 0x49e6b9) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x119405e0900ebe1, 0x300, 0xc, 0x9baf000) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7236e27, 0x3b81}, 0x0, 0xc03f81b478) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc03f81b200, {0xffffffffffffff15, 0xc0940b25b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc03f81b200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc03f81b200, {0xc031a29680, 0x32, 0x7f}, 0x13, 0x15) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc03f81b200, {0xc02e764480, 0x32, 0x7f}, {0xc031a29680, 0x51c9860, 0xc031a29680}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc03f81b200, {0xc02e764480, 0x32, 0x79}, {0xc031a29680, 0x32, 0x7f}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091603298, {0xc02e764480, 0x32, 0x79}, {0xc031a29680, 0x32, 0x7f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091601bc0, {0xc031a29680, 0x7f, 0x79}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091601bc0, {0xc02e764480, 0x7f, 0xc031a29680}, {0xc031a29680, 0xc02ef5cd80, 0x31}, 0x7d) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091601900, {0xc031a29680, 0x32, 0x7f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc077483328, {{0xc02ef5cd80, 0x31, 0x7d}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc049d87100, {{0xc0587120c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc038182c00, {0x63c85b8, 0xc042f176e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc042f176e0}, {0x64a6560, 0xc049d87100}, {0xc0587120c0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc042f176e0}, {0x6466ed8, 0xc077483080}, {0xc0587120c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc064ac0b58}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc061baad00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020798000, {0x63c85b8, 0xc042f176e0}, 0xc061baad00, 0xc048082fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020798000, {0x63c85b8, 0xc042f176e0}, 0xc061baad00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042f17650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042f17650}, {0xc0492c88f0, 0x660089}, 0xc0492c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042f17650}, {0xc00102e4e0}, 0xc061baac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042f175f0}, 0xc061baac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc038bba690}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810599 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc051005710}, 0xc047405278, 0x1, 0xc0287e10a8, 0x519bbc, 0xc0457b5020, 0xc0626b0180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc051005710}, 0x120, 0xc0287e1230, 0x457b4fc0, 0x0, 0x660069, 0xc0457b5020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc051005710}, 0xc0457b4fc0, {{{{0x0, 0x0}, {0xc08a77f680, 0x2c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc051005710}, 0x6352410, 0x20fd3360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc051005710}, {0xc0457b4ea0, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc051005710}, 0xc05292fea0, 0xc0287e19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc051005710}, 0x660089, {0xc0457b4ea0, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc051005710}, 0xc08ce1c200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc051005680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051005680}, {0xc0287e33f8, 0x660089}, 0xc0287e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc051005680}, {0x0}, 0xc08ce1c000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0510055f0}, 0xc08ce1c000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}, 0xc0213c0280, 0xc03b27a960) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}, 0x3, 0xc0635b25c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0287e3980}}}, {0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}, 0xc0213c0280, 0xc0635b25c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0510055f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0510055f0}, {0x40, 0xc0287e3b70}, 0xc0287e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0510055f0}, {0x4e9d7a0, 0xc08ce1c000}, 0xc0213c0280, 0xc03b27a960) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0510055f0}, 0xc0ae98a480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0457b4d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0457b4d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219796962 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4d868, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07318f400, 0xc00f57e000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07318f400, {0xc00f57e000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07318f400, {0xc00f57e000, 0x735, 0xc064d4c280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc036ce9e10, {0xc00f57e000, 0x7eda79, 0xc00f57e005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc07653a0c0, {0xc00f57e000, 0x3, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b26910f8, {0xc00f57e000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc084688cf8, {0x631b900, 0xc0b26910f8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc084688a80, {0x7fe42ab59310, 0xc07653a0c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc084688a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc084688a80, {0xc00cbdf000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc05944d2d8, {0xc00cbdf000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc072e0eb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc072e0eb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc072e0f0e0, {0x634aac0, 0xc072e0eb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc072e0ea80, 0xc0594d5e70, 0xc081bc542b, 0xc081bc542d, {0x635bec0, 0xc06b2e0360}, 0xc000d6ae00, 0xc013e824c8, 0xc081bc5429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc072e0ea80, {0x63c85b8, 0xc071fea0f0}, 0xc0594d5e60, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0508c703e, 0x8}}, 0x0, 0xc02bb31f20, 0xc071fea090, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc071fea0f0}, {0x643d810, 0xc07653a0c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0419becc0}, {0x643d810, 0xc07653a0c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0419becc0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803945 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc00fb3def0, {0x63c85b8, 0xc0164cb9b0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc013996000, {0x63c85b8, 0xc0164cb9b0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc013996000, {0x63c8510, 0xc066c88a80}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc013996000, {0x63c8510, 0xc066c88a80}, {0x63f7db8, 0x9bf6f48}, 0xc08f8f1088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc013996000, {0x63c85b8, 0xc03515ccf0}, {{0x63f7db8, 0x9bf6f48}, {0xc07297341e, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc013996000, {0x63c85b8, 0xc03515ccf0}, {{0x63f7db8, 0x9bf6f48}, {0xc07297341e, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc07297341e, 0x6}, 0x0, 0x0}, {0x2032f548, 0xedb7a834f, 0x0}, {0x2032ff1f, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc013996000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc013996000, {0x63c8510, 0xc059ea7bc0}, 0xc076029dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01dc59e00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc059ea7bc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01dc59e00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801410 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f46d000, {0x63c85b8, 0xc0619bde60}, 0xc097298b00, 0xc04e3be280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f46d000, {0x63c85b8, 0xc0619bde60}, 0xc097298b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b5e7f26, 0x0, 0x0}, 0xc00d4f6730, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b5e7f26, 0x0, 0x0}, 0xc00d4f6730, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0619bddd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0619bddd0}, {0xc0766cf918, 0x660089}, 0xc0766cf920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0619bddd0}, {0xc00102e4e0}, 0xc097298a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0619bdd70}, 0xc097298a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc03a9b2f60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0619bdce0}, {0xc00dfd36b0, 0x24}, 0x46c7b3d46, 0xc0387f9d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x10, 0xc000510478}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0619bdce0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806056 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc07d473350, {0x63c85b8, 0xc06f6cadb0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc06f6cadb0}, {0x4f67ad2, 0x10beec0}, 0xc088194fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219810154 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0357c0, {0x63c85b8, 0xc017d58c30}, 0xc04cebb818, 0x1, 0xc01cf090a8, 0x519bbc, 0xc05a72b6e0, 0xc065776600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc017d58c30}, 0x120, 0xc01cf09230, 0x5a72b680, 0x0, 0x660069, 0xc05a72b6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0357c0, {0x63c85b8, 0xc017d58c30}, 0xc05a72b680, {{{{0x0, 0x0}, {0xc060559440, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc017d58c30}, 0x63526b0, 0x43391f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd770, {0x63c85b8, 0xc017d58c30}, {0xc05a72b560, {0x174301106bafa965, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd770, {0x63c85b8, 0xc017d58c30}, 0xc02a1a3860, 0xc01cf099f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc017d58c30}, 0x660089, {0xc05a72b560, {0x174301106bafa965, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc017d58c30}, 0xc04bcde000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bafa965, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bafa965, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc017d58ba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc017d58ba0}, {0xc01cf0b3f8, 0x660089}, 0xc01cf0b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc017d58ba0}, {0x0}, 0xc0551b3e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc017d58b10}, 0xc0551b3e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}, 0xc01696d320, 0xc06ea00300) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}, 0x3, 0xc00ff07d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01cf0b980}}}, {0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}, 0xc01696d320, 0xc00ff07d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc017d58b10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc017d58b10}, {0x40, 0xc01cf0bb70}, 0xc01cf0bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc017d58b10}, {0x4e9d7a0, 0xc0551b3e00}, 0xc01696d320, 0xc06ea00300) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc017d58b10}, 0xc04ebfb6e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a72b320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a72b320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802662 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021d20400, {0x63c85b8, 0xc015b5bec0}, 0xc088b64d00, 0xc022383c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021d20400, {0x63c85b8, 0xc015b5bec0}, 0xc088b64d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024b4cd54, 0x0, 0x0}, 0xc0436340d0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024b4cd54, 0x0, 0x0}, 0xc0436340d0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc015b5be30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015b5be30}, {0xc076675918, 0x660089}, 0xc076675920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc015b5be30}, {0xc00102e4e0}, 0xc088b64c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc015b5bdd0}, 0xc088b64c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc068483500}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04e498ba0}, {0xc00dfd36b0, 0x24}, 0x46c7c0ef8, 0xc01728dd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc082aa9050}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04e498ba0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812696 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801384 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0976f4548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0976f45b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07d3aacf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04b474380, {0xc05af9ab60, 0x2b, 0xd0}, {0xc0bd2b48c0, 0x2b, 0x6d}, 0x1, 0xac) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07d3aab88, {0xc05af9ab60, 0x1, 0x6d}, {0xc0bd2b48c0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d3aab88, {0xc05af9ab60, 0x2b, 0xd0}, {0xc0bd2b48c0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d3aa2c0, {0xc0bd2b48c0, 0x6d, 0xd0}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d3aa2c0, {0xc05af9ab60, 0x2b, 0xc0bd2b48c0}, {0xc0bd2b48c0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d3aa000, {0xc0bd2b48c0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0ba54d608, {{0xc03e50f2c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc045890700, {{0xc03e50f2c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04b33a800, {0x63c85b8, 0xc057b0fad0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc057b0fad0}, {0x64a6560, 0xc045890700}, {0xc03e50f2c0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc057b0fad0}, {0x6466ed8, 0xc0ba54d600}, {0xc03e50f2c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04459e3d8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0474a0600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c90000, {0x63c85b8, 0xc057b0fad0}, 0xc0474a0600, 0xc05b44e0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c90000, {0x63c85b8, 0xc057b0fad0}, 0xc0474a0600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057b0fa40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057b0fa40}, {0xc0d68328f0, 0x660089}, 0xc0d68328f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057b0fa40}, {0xc00102e4e0}, 0xc03399be00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057b0f9e0}, 0xc03399be00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc017f71f80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804581 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0489f06c0}, 0xc02a22a400, 0xc02c4b0140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0489f06c0}, 0xc02a22a400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110356c23ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110356c23ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0489f0630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0489f0630}, {0xc052ef93f8, 0x660089}, 0xc052ef9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0489f0630}, {0xc0015221e0}, 0xc051cf1e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0489f0540}, 0xc051cf1e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}, 0xc054286de0, 0xc048fc6480) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}, 0x3, 0xc042ee5080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc052ef9980}}}, {0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}, 0xc054286de0, 0xc042ee5080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0489f0540}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0489f0540}, {0x40, 0xc052ef9b70}, 0xc052ef9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0489f0540}, {0x4e9d7a0, 0xc051cf1e00}, 0xc054286de0, 0xc048fc6480) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0489f0540}, 0xc0ac7851a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc063f047e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc063f047e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810156 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc049eb6570}, 0xc04bcdee00, 0xc06191afa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc049eb6570}, 0xc04bcdee00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106be8631b, 0x0, 0x0}, 0xc04bda53d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106be8631b, 0x0, 0x0}, 0xc04bda53d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc049eb64e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049eb64e0}, {0xc01aafb3f8, 0x660089}, 0xc01aafb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc049eb64e0}, {0x0}, 0xc04bcdeb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc049eb6450}, 0xc04bcdeb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}, 0xc06fc90340, 0xc06ea00ed0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}, 0x3, 0xc03b504b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aafb980}}}, {0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}, 0xc06fc90340, 0xc03b504b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc049eb6450}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049eb6450}, {0x40, 0xc01aafbb70}, 0xc01aafbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc049eb6450}, {0x4e9d7a0, 0xc04bcdeb00}, 0xc06fc90340, 0xc06ea00ed0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc049eb6450}, 0xc01d7efda0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0681f7320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0681f7320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801826 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc068944ab0}, 0xc04b888a00, 0xc050935e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc068944ab0}, 0xc04b888a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d2e34ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d2e34ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc068944a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc068944a20}, {0xc055fd33f8, 0x660089}, 0xc055fd3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc068944a20}, {0x32}, 0xc07e2ebf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc068944990}, 0xc07e2ebf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}, 0xc07b570140, 0xc09fd462b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}, 0x3, 0xc04626c200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055fd3980}}}, {0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}, 0xc07b570140, 0xc04626c200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc068944990}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc068944990}, {0x40, 0xc055fd3b70}, 0xc055fd3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc068944990}, {0x4e9d7a0, 0xc07e2ebf00}, 0xc07b570140, 0xc09fd462b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc068944990}, 0xc05fffdc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc023b33d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc023b33d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219114493 [runnable]: internal/poll.runtime_pollWait(0x7fe456a4dfe8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0272da480, 0xc00c40f800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0272da480, {0xc00c40f800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0272da480, {0xc00c40f800, 0x735, 0xc040c9c500}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc046cec5a8, {0xc00c40f800, 0x7eda79, 0xc00c40f805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc023f13440, {0xc00c40f800, 0x35318b4, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc01c51c018, {0xc00c40f800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc01f6ad778, {0x631b900, 0xc01c51c018}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc01f6ad500, {0x7fe42ab59310, 0xc023f13440}, 0x60) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc01f6ad500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc01f6ad500, {0xc079462000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02ddbede0, {0xc079462000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0866cab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0866cab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0866cb0e0, {0x634aac0, 0xc0866cab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0866caa80, 0xc07ac38880, 0xc09be3142b, 0xc09be3142d, {0x635bec0, 0xc026eb7ef0}, 0xc000d6ae00, 0xc0418c7dd8, 0xc09be31429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0866caa80, {0x63c85b8, 0xc02555e090}, 0xc07ac38870, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc015f2f058, 0x8}}, 0x0, 0xc02555e030, 0xc02555e060, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02555e090}, {0x643d810, 0xc023f13440}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03b17f410}, {0x643d810, 0xc023f13440}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03b17f410}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808729 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc0823eca20}, 0xc0310fb2d8, 0x1, 0xc005aed0a8, 0x519bbc, 0xc02cc48420, 0xc07d44b8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0823eca20}, 0x1e0, 0xc005aed230, 0x2cc483c0, 0x0, 0x660069, 0xc02cc48420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc0823eca20}, 0xc02cc483c0, {{{{0x0, 0x0}, {0xc053ff9b00, 0x55}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0823eca20}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc0823eca20}, {0xc067a5d560, {0x174301105f9638ed, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc0823eca20}, 0xc0554d7180, 0xc005aed9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0823eca20}, 0x660089, {0xc067a5d560, {0x174301105f9638ed, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0823eca20}, 0xc056380900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105f9638ed, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105f9638ed, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0823ec990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0823ec990}, {0xc005aef3f8, 0x660089}, 0xc005aef400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0823ec990}, {0x40e8faff2ddd4c3a}, 0xc056380500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0823ec900}, 0xc056380500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}, 0xc03f1eb800, 0xc06f94c528) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}, 0x3, 0xc088af9000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005aef980}}}, {0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}, 0xc03f1eb800, 0xc088af9000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0823ec900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0823ec900}, {0x40, 0xc005aefb70}, 0xc005aefab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0823ec900}, {0x4e9d7a0, 0xc056380500}, 0xc03f1eb800, 0xc06f94c528) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0823ec900}, 0xc0a41ea720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0848e1c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0848e1c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802193 [runnable]: sync.runtime_SemacquireMutex(0xc00074b2c0, 0x0, 0x11840a5) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).Lock(0xc091d56338) GOROOT/src/sync/rwmutex.go:116 +0x71 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074b2b0, 0x11869e5, 0x7fe1a3f69800, 0x7f87, 0x7fd753c70e80) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc025717f80, 0x7fe1a3f69801, 0x7fe1a3f69820, 0x426c, 0x4271) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc025859600, {0x1c38fe6, 0x426c}, 0x0, 0xc0552c9d78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0552c9b00, {0xffffffffffffff15, 0xc091d565c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0552c9b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0552c9b00, {0xc0830f7ab0, 0x2b, 0x6d}, 0x13, 0x9f) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0552c9b00, {0xc024910300, 0x2b, 0x6d}, {0xc0830f7ab0, 0x51c9860, 0xc0830f7ab0}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0552c9b00, {0xc024910300, 0x2b, 0x80}, {0xc0830f7ab0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093207e98, {0xc024910300, 0x2b, 0x80}, {0xc0830f7ab0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0932067c0, {0xc0830f7ab0, 0x6d, 0x80}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0932067c0, {0xc024910300, 0x2b, 0xc0830f7ab0}, {0xc0830f7ab0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093206500, {0xc0830f7ab0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc076952b08, {{0xc047c357d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc00c23a600, {{0xc047c357d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05315c800, {0x63c85b8, 0xc0688e2b40}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0688e2b40}, {0x64a6560, 0xc00c23a600}, {0xc047c357d0, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0688e2b40}, {0x6466ed8, 0xc076952b00}, {0xc047c357d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01fb440c0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc09c55f000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ee1400, {0x63c85b8, 0xc0688e2b40}, 0xc09c55f000, 0xc0682a7720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ee1400, {0x63c85b8, 0xc0688e2b40}, 0xc09c55f000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0688e2ab0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0688e2ab0}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0688e2ab0}, {0xc00102e4e0}, 0xc09c55ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0688e2a50}, 0xc09c55ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0632205a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812523 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811304 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc05f8fede0}, 0xc06859b038, 0x1, 0xc0064e10a8, 0x519bbc, 0xc08c3b3500, 0xc073ea76e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc05f8fede0}, 0x1e0, 0xc0064e1230, 0x8c3b34a0, 0x0, 0x660069, 0xc08c3b3500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc05f8fede0}, 0xc08c3b34a0, {{{{0x0, 0x0}, {0xc0504e4900, 0x114}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05f8fede0}, 0x6352260, 0x4207e738) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc05f8fede0}, {0xc05073c360, {0x1743011074cc4cba, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc05f8fede0}, 0xc01daad7c0, 0xc0064e19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05f8fede0}, 0x660089, {0xc05073c360, {0x1743011074cc4cba, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc05f8fede0}, 0xc0862ad600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011074cc4cba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011074cc4cba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05f8fed50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f8fed50}, {0xc0064e33f8, 0x660089}, 0xc0064e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05f8fed50}, {0xd837b1f616008801}, 0xc0862ad400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05f8fecc0}, 0xc0862ad400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}, 0xc00cd96f60, 0xc03fdf5308) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}, 0x3, 0xc042b0b980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0064e3980}}}, {0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}, 0xc00cd96f60, 0xc042b0b980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05f8fecc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f8fecc0}, {0x40, 0xc0064e3b70}, 0xc0064e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05f8fecc0}, {0x4e9d7a0, 0xc0862ad400}, 0xc00cd96f60, 0xc03fdf5308) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05f8fecc0}, 0xc0396c8ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05073c240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05073c240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219653820 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1cb0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05beac380, 0xc028cb4000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05beac380, {0xc028cb4000, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05beac380, {0xc028cb4000, 0x739, 0xc0cc547e00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc02f62f4e0, {0xc028cb4000, 0x7eda79, 0xc028cb4005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02e124360, {0xc028cb4000, 0x0, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b87c8b10, {0xc028cb4000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0cec53af8, {0x631b900, 0xc0b87c8b10}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0cec53880, {0x7fe42ab59310, 0xc02e124360}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0cec53880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0cec53880, {0xc0ba755000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc040a85ad0, {0xc0ba755000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0789e5588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0789e5588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0789e5b60, {0x634aac0, 0xc0789e5588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0789e5500, 0xc04b0194d0, 0xc0847cb42b, 0xc0847cb42d, {0x635bec0, 0xc03db4d170}, 0xc000d6ae00, 0xc04244f458, 0xc0847cb429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0789e5500, {0x63c85b8, 0xc04b2c22d0}, 0xc04b0194c0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc033ba504e, 0xb}}, 0x0, 0xc04b2c2270, 0xc04b2c22a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04b2c22d0}, {0x643d810, 0xc02e124360}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03af96a80}, {0x643d810, 0xc02e124360}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03af96a80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808612 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc061dce000, {0x63c85b8, 0xc045f888d0}, 0xc066826700, 0xc071f3bb80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc045f888d0}, 0xc066826700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b709713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b709713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045f88840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045f88840}, {0xc00602f3f8, 0x660089}, 0xc00602f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045f88840}, {0x3feffffee1276011}, 0xc066826300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045f887b0}, 0xc066826300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}, 0xc0118dd1e0, 0xc00172d5a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}, 0x3, 0xc06b743b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00602f980}}}, {0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}, 0xc0118dd1e0, 0xc06b743b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045f887b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045f887b0}, {0x40, 0xc00602fb70}, 0xc00602fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045f887b0}, {0x4e9d7a0, 0xc066826300}, 0xc0118dd1e0, 0xc00172d5a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045f887b0}, 0xc024319260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0646bbb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0646bbb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809859 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc020055fb0}, 0xc02710d920, 0x1, 0xc00ebe90a8, 0x519bbc, 0xc0822d2a80, 0xc0c9892720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc020055fb0}, 0x120, 0xc00ebe9230, 0x822d2a20, 0x0, 0x660069, 0xc0822d2a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc020055fb0}, 0xc0822d2a20, {{{{0x0, 0x0}, {0xc054807680, 0xea}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc020055fb0}, 0x6352410, 0x3102f1c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc020055fb0}, {0xc0822d2900, {0x1743011067081cd9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc020055fb0}, 0xc0736d1360, 0xc00ebe99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc020055fb0}, 0x660089, {0xc0822d2900, {0x1743011067081cd9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc020055fb0}, 0xc05b35eb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067081cd9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067081cd9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020055e30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020055e30}, {0xc00ebeb3f8, 0x660089}, 0xc00ebeb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020055e30}, {0x7fe347d135e0}, 0xc05b35e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020055cb0}, 0xc05b35e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}, 0xc0631ce980, 0xc066ab56f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}, 0x3, 0xc0151b6380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ebeb980}}}, {0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}, 0xc0631ce980, 0xc0151b6380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020055cb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020055cb0}, {0x40, 0xc00ebebb70}, 0xc00ebebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020055cb0}, {0x4e9d7a0, 0xc05b35e800}, 0xc0631ce980, 0xc066ab56f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020055cb0}, 0xc06e3ba2a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0822d27e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0822d27e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812443 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0185643c0, {0x63c85b8, 0xc073d044b0}, 0xc0442e04c8, 0x1, 0xc004d530a8, 0x519bbc, 0xc0c9977040, 0xc072298ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0667e85a0, {0x63c85b8, 0xc073d044b0}, 0xc0667e85d0, 0xc004d531a8, 0x30, 0x40, 0xffffffffffffffff, 0xc0c9977040, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0185643c0, {0x63c85b8, 0xc073d044b0}, 0xc0c9976e00, {{{{0x0, 0x0}, {0xc0564f5e60, 0x7}}, {{0xc051e3ca20, 0x6}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073d044b0}, 0x63521a0, 0x3abf6838) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0185322d0, {0x63c85b8, 0xc073d044b0}, {0xc0137dafc0, {0x174301100f6ceddc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0185322d0, {0x63c85b8, 0xc073d044b0}, 0xc065f2f9a0, 0xc004d539f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073d044b0}, 0x660089, {0xc0137dafc0, {0x174301100f6ceddc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018570c00, {0x63c85b8, 0xc073d044b0}, 0xc0411c6300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100f6ceddc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100f6ceddc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073d043c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d043c0}, {0xc004d553f8, 0x660089}, 0xc004d55400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073d043c0}, {0xc0015221e0}, 0xc0411c6100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073d042d0}, 0xc0411c6100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}, 0xc074682aa0, 0xc018c978a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}, 0x3, 0xc0746f4f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004d55980}}}, {0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}, 0xc074682aa0, 0xc0746f4f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073d042d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d042d0}, {0x40, 0xc004d55b70}, 0xc004d55ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073d042d0}, {0x4e9d7a0, 0xc0411c6100}, 0xc074682aa0, 0xc018c978a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073d042d0}, 0xc070e2e900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0137daea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0137daea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806498 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64000, {0x63c85b8, 0xc07c5d9bf0}, 0xc08fa4a480, 0x1, 0xc00a1810a8, 0x519bbc, 0xc023bf7ec0, 0xc058d0b380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07c5d9bf0}, 0x120, 0xc00a181230, 0x23bf7e60, 0x0, 0x660069, 0xc023bf7ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64000, {0x63c85b8, 0xc07c5d9bf0}, 0xc023bf7e60, {{{{0x0, 0x0}, {0xc041f937a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07c5d9bf0}, 0x6352230, 0x17e2f150) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a588c0, {0x63c85b8, 0xc07c5d9bf0}, {0xc023bf7d40, {0x174301104e0376bd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a588c0, {0x63c85b8, 0xc07c5d9bf0}, 0xc04ff24780, 0xc00a1819f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07c5d9bf0}, 0x660089, {0xc023bf7d40, {0x174301104e0376bd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016ba0000, {0x63c85b8, 0xc07c5d9bf0}, 0xc0676a6e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e0376bd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e0376bd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c5d9b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c5d9b60}, {0xc00a1833f8, 0x660089}, 0xc00a183400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c5d9b60}, {0xc0015221e0}, 0xc0676a6c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c5d9ad0}, 0xc0676a6c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}, 0xc08a538200, 0xc090f95068) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}, 0x3, 0xc07cec8b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a183980}}}, {0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}, 0xc08a538200, 0xc07cec8b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c5d9ad0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c5d9ad0}, {0x40, 0xc00a183b70}, 0xc00a183ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c5d9ad0}, {0x4e9d7a0, 0xc0676a6c00}, 0xc08a538200, 0xc090f95068) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c5d9ad0}, 0xc06607bce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc023bf79e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc023bf79e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802035 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0401b7f50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 206962029 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc017913790, 0x68c) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc06c7b3500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc06c7b3b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc07a06e000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc07a06e000, {0x63c8510, 0xc0179137c0}, 0xc085b75dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc082e4a520) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0179137c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc082e4a520) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812745 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806916 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc03fcb2b00, {0x63c85b8, 0xc031793ec0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc00d096c00, {0x63c85b8, 0xc031793ec0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc00d096c00, {0x63c8510, 0xc00c611680}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc00d096c00, {0x63c8510, 0xc00c611680}, 0xc057576590, {0x7fe42b790cf0, 0xc0aadde6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00d096c00, {0x63c85b8, 0xc020b7d3b0}, {{0x63f88f8, 0xc057576590}, {0xc09da3ba1c, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00d096c00, {0x63c85b8, 0xc020b7d3b0}, {{0x63f88f8, 0xc057576590}, {0xc09da3ba1c, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc057576590}, {0xc09da3ba1c, 0x23}, 0x0, 0x0}, {0xdb119ad, 0xedb7a8350, 0x0}, {0xdb11c9b, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00d096c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00d096c00, {0x63c8510, 0xc08fb47280}, 0xc082d13dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc091aa3f80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc08fb47280}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc091aa3f80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219807580 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc06e374690}, 0xc0cd349350, 0x1, 0xc005c9f0a8, 0x519bbc, 0xc073921c80, 0xc07c67faa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06e374690}, 0x1e0, 0xc005c9f230, 0x73921c20, 0x0, 0x660069, 0xc073921c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc06e374690}, 0xc073921c20, {{{{0x0, 0x0}, {0xc06040ad80, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06e374690}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc06e374690}, {0xc073d85680, {0x174301105981709e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc06e374690}, 0xc02a659d60, 0xc005c9f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06e374690}, 0x660089, {0xc073d85680, {0x174301105981709e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc06e374690}, 0xc04acb2a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105981709e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105981709e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e374600}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e374600}, {0xc005ca13f8, 0x660089}, 0xc005ca1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e374600}, {0x3feffe1328e8ab1a}, 0xc04acb2100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e374570}, 0xc04acb2100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}, 0xc03d581100, 0xc04ddb2270) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}, 0x3, 0xc0762ecb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ca1980}}}, {0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}, 0xc03d581100, 0xc0762ecb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06e374570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e374570}, {0x40, 0xc005ca1b70}, 0xc005ca1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06e374570}, {0x4e9d7a0, 0xc04acb2100}, 0xc03d581100, 0xc04ddb2270) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06e374570}, 0xc0424ce0c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc073d84ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc073d84ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809060 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc025911260}, 0xc0506a7e78, 0x1, 0x0, 0x1, 0xc066880960, 0xc06c2eb5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc025911260}, 0x120, 0xc0200f0728, 0x66880900, 0x0, 0x660069, 0xc066880960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc025911260}, 0xc066880900, {{{{0x0, 0x0}, {0xc012538ea0, 0xb}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc025911260}, 0x6352410, 0x3f8ea7e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc025911260}, {0xc06d9b9560, {0x1743011060f4f972, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc025911260}, 0xc03d3c99a0, 0xc06f204ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc025911260}, 0x660089, {0xc06d9b9560, {0x1743011060f4f972, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc025911260}, 0xc041b4c700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011060f4f972, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011060f4f972, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0259110e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0259110e0}, {0xc0025148f0, 0x660089}, 0xc0025148f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0259110e0}, {0xc00102e4e0}, 0xc041b4c600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc025910de0}, 0xc041b4c600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04701bb00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809378 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc020b7cab0}, 0xc03f325d40, 0x1, 0xc01f4cf0a8, 0x519bbc, 0xc05564d6e0, 0xc06e300ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc020b7cab0}, 0x1e0, 0xc01f4cf230, 0x5564d680, 0x0, 0x660069, 0xc05564d6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc020b7cab0}, 0xc05564d680, {{{{0x0, 0x0}, {0xc04d19e6c0, 0xac}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc03f703c20, {0x63c85b8, 0xc020b7cab0}, 0xc054155950, 0x1f4cf428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc020b7cab0}, {0xc0480d46c0, {0x174301106386cfa2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc020b7cab0}, 0xc075976fa0, 0xc01f4cf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc020b7cab0}, 0x660089, {0xc0480d46c0, {0x174301106386cfa2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc020b7cab0}, 0xc060ccaa00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106386cfa2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106386cfa2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020b7c990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020b7c990}, {0xc01f4d13f8, 0x660089}, 0xc01f4d1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020b7c990}, {0xc0015221e0}, 0xc060cca700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020b7c810}, 0xc060cca700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}, 0xc01c863120, 0xc059579ad0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}, 0x3, 0xc0065897c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f4d1980}}}, {0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}, 0xc01c863120, 0xc0065897c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020b7c810}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020b7c810}, {0x40, 0xc01f4d1b70}, 0xc01f4d1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020b7c810}, {0x4e9d7a0, 0xc060cca700}, 0xc01c863120, 0xc059579ad0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020b7c810}, 0xc0275348a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0480d45a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0480d45a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801644 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc03a8d1fb0}, 0xc0d35fe300, 0x1, 0xc03f93f0a8, 0x519bbc, 0xc0479c6f00, 0xc091e9f740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc060791440, {0x63c85b8, 0xc03a8d1fb0}, 0xc0b89f9320, 0xc03f93f1a8, 0x479c6f60, 0x0, 0x1, 0xc0479c6f00, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc03a8d1fb0}, 0xc0479c6ea0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc085139200, 0x39}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc03a8d1fb0}, 0x63523b0, 0x65636140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc03a8d1fb0}, {0x0, {0x174301101f039359, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc03a8d1fb0}, 0xc049b105a0, 0xc03f93f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03a8d1fb0}, 0x660089, {0x0, {0x174301101f039359, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03a8d1fb0}, 0xc04cf0c200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101f039359, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101f039359, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301101f039359, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03a8d1f20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a8d1f20}, {0xc03f9413f8, 0x660089}, 0xc03f941400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03a8d1f20}, {0xc0015221e0}, 0xc04cf0c000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a8d1e90}, 0xc04cf0c000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}, 0xc05f6f0b20, 0xc09b894600) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}, 0x3, 0xc0c0170c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03f941980}}}, {0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}, 0xc05f6f0b20, 0xc0c0170c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03a8d1e90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a8d1e90}, {0x40, 0xc03f941b70}, 0xc03f941ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03a8d1e90}, {0x4e9d7a0, 0xc04cf0c000}, 0xc05f6f0b20, 0xc09b894600) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03a8d1e90}, 0xc0c0552000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0479c6d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0479c6d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802187 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc0cef7dec0}, 0xc02febec60, 0x1, 0x0, 0x1, 0xc06fcd8600, 0xc09e407c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0cef7dec0}, 0x120, 0xc0588ee728, 0x6fcd85a0, 0x0, 0x660069, 0xc06fcd8600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc0cef7dec0}, 0xc06fcd85a0, {{{{0x0, 0x0}, {0xc088ff3c20, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0cef7dec0}, 0x6352410, 0x489dd1e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc0cef7dec0}, {0xc06fcd8120, {0x1743011020c2c92c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc0cef7dec0}, 0xc022f4a820, 0xc04c2f2ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0cef7dec0}, 0x660089, {0xc06fcd8120, {0x1743011020c2c92c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc0cef7dec0}, 0xc040fcca00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020c2c92c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020c2c92c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0cef7de30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0cef7de30}, {0xc0036f68f0, 0x660089}, 0xc0036f68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0cef7de30}, {0xc00102e4e0}, 0xc040fcc800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0cef7ddd0}, 0xc040fcc800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0cef7daa0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811861 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc0654c0750}, 0xc056b27ec0, 0x1, 0xc00bc0b0a8, 0x519bbc, 0xc039070ba0, 0xc01ada2ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0654c0750}, 0x1e0, 0xc00bc0b230, 0x39070b40, 0x0, 0x660069, 0xc039070ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc0654c0750}, 0xc039070b40, {{{{0x0, 0x0}, {0xc0a626ed80, 0x6c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0654c0750}, 0x63521a0, 0x47c52160) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc0654c0750}, {0xc059511320, {0x174301107cc7b567, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc0654c0750}, 0xc050935ae0, 0xc00bc0b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0654c0750}, 0x660089, {0xc059511320, {0x174301107cc7b567, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0654c0750}, 0xc06e40e300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107cc7b567, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107cc7b567, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0654c06c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0654c06c0}, {0xc00bc0d3f8, 0x660089}, 0xc00bc0d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0654c06c0}, {0x7fe3ba7617b8}, 0xc06e40e100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0654c0630}, 0xc06e40e100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}, 0xc0428fd3e0, 0xc03f861998) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}, 0x3, 0xc0450f8300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00bc0d980}}}, {0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}, 0xc0428fd3e0, 0xc0450f8300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0654c0630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0654c0630}, {0x40, 0xc00bc0db70}, 0xc00bc0dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0654c0630}, {0x4e9d7a0, 0xc06e40e100}, 0xc0428fd3e0, 0xc03f861998) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0654c0630}, 0xc0bac64f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0595110e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0595110e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809210 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc05375b710}, 0xc04b483b00, 0xc053fbd220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc05375b710}, 0xc04b483b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106415dbca, 0x0, 0x0}, 0xc04d5824e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106415dbca, 0x0, 0x0}, 0xc04d5824e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05375b680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05375b680}, {0xc0127f53f8, 0x660089}, 0xc0127f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05375b680}, {0x0}, 0xc04b483900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05375b5f0}, 0xc04b483900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}, 0xc0761ebb20, 0xc049a03a70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}, 0x3, 0xc070418400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0127f5980}}}, {0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}, 0xc0761ebb20, 0xc070418400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05375b5f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05375b5f0}, {0x40, 0xc0127f5b70}, 0xc0127f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05375b5f0}, {0x4e9d7a0, 0xc04b483900}, 0xc0761ebb20, 0xc049a03a70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05375b5f0}, 0xc056cf3e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0670a2120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0670a2120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809304 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc039fcd680}, 0xc020bd5458, 0x1, 0xc0268af0a8, 0x519bbc, 0xc06cf60de0, 0xc0750c5140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc039fcd680}, 0x120, 0xc0268af230, 0x6cf60d80, 0x0, 0x660069, 0xc06cf60de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc039fcd680}, 0xc06cf60d80, {{{{0x0, 0x0}, {0xc046a7c480, 0x5b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc039fcd680}, 0x6352410, 0x43391440) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc039fcd680}, {0xc06cf60c60, {0x174301105af607c5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc039fcd680}, 0xc0389f0be0, 0xc0268af9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc039fcd680}, 0x660089, {0xc06cf60c60, {0x174301105af607c5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc039fcd680}, 0xc09d60af00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105af607c5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105af607c5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039fcd5f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039fcd5f0}, {0xc0268b13f8, 0x660089}, 0xc0268b1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039fcd5f0}, {0x7fe3f1e3ec20}, 0xc09d60ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039fcd560}, 0xc09d60ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}, 0xc0729cbfa0, 0xc0452207b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}, 0x3, 0xc040ec5c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0268b1980}}}, {0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}, 0xc0729cbfa0, 0xc040ec5c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc039fcd560}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039fcd560}, {0x40, 0xc0268b1b70}, 0xc0268b1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc039fcd560}, {0x4e9d7a0, 0xc09d60ad00}, 0xc0729cbfa0, 0xc0452207b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc039fcd560}, 0xc043cdd3e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06cf60b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06cf60b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806529 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc08c6ca000}, 0xc08d7faa68, 0x1, 0xc00ec216b8, 0x519bbc, 0xc089b885a0, 0xc00f0314a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc07b822c60, {0x63c85b8, 0xc08c6ca000}, 0xc04b78e000, 0xc00ec217b8, 0x89b885a0, 0x0, 0x2, 0xc089b885a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc08c6ca000}, 0xc089b88540, {{{{0xc042b3b680, 0x3d}, {0xc07b98c480, 0x5b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08c6ca000}, 0x63525f0, 0x1fae7468) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc08c6ca000}, {0x0, {0x174301104fcd81d5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc08c6ca000}, 0xc049d78500, 0xc00ec22000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08c6ca000}, 0x660089, {0x0, {0x174301104fcd81d5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc08c6ca000}, 0xc03e663a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104fcd81d5, 0x0, 0x0}, 0xc0896cafb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104fcd81d5, 0x0, 0x0}, 0xc0896cafb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08ba0df50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08ba0df50}, {0xc082d11a08, 0x660089}, 0xc082d11a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08ba0df50}, {0xc00102e4e0}, 0xc03e663900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08ba0def0}, 0xc03e663900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc08ba0de60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc08ba0de60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807229 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1da0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc08471e300, 0xc07c9fe800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc08471e300, {0xc07c9fe800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc08471e300, {0xc07c9fe800, 0x735, 0xc0511b5a40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc05449bf10, {0xc07c9fe800, 0x7eda79, 0xc07c9fe805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02269cde0, {0xc07c9fe800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2690d98, {0xc07c9fe800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc07901be78, {0x631b900, 0xc0b2690d98}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc07901bc00, {0x7fe42ab59310, 0xc02269cde0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc07901bc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc07901bc00, {0xc06eaf4000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06605aeb8, {0xc06eaf4000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06ec08b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06ec08b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06ec090e0, {0x634aac0, 0xc06ec08b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06ec08a80, 0xc038af5310, 0xc078c5b42b, 0xc078c5b42d, {0x635bec0, 0xc0156fb050}, 0xc000d6ae00, 0xc090690a88, 0xc078c5b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06ec08a80, {0x63c85b8, 0xc082fb6de0}, 0xc038af5300, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc099495021, 0x8}}, 0x0, 0xc082fb6d80, 0xc082fb6db0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc082fb6de0}, {0x643d810, 0xc02269cde0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03e83a360}, {0x643d810, 0xc02269cde0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03e83a360}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809867 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0233b7980}, 0xc07ced3500, 0xc03ae9dcc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0233b7980}, 0xc07ced3500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105ee9eb70, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105ee9eb70, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0233b7710}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0233b7710}, {0xc01e9d73f8, 0x660089}, 0xc01e9d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0233b7710}, {0x7fe42923de40}, 0xc07ced3200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0233b7650}, 0xc07ced3200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}, 0xc05f82c200, 0xc067f104e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}, 0x3, 0xc099165f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01e9d7980}}}, {0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}, 0xc05f82c200, 0xc099165f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0233b7650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0233b7650}, {0x40, 0xc01e9d7b70}, 0xc01e9d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0233b7650}, {0x4e9d7a0, 0xc07ced3200}, 0xc05f82c200, 0xc067f104e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0233b7650}, 0xc03179e780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07dd36000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07dd36000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807928 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc077516390}, 0xc0479f08e8, 0x1, 0x0, 0x1, 0xc073140840, 0xc0b7661380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077516390}, 0x120, 0xc008878728, 0x731407e0, 0x0, 0x660069, 0xc073140840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc077516390}, 0xc0731407e0, {{{{0x0, 0x0}, {0xc0ada1b9e0, 0x14}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077516390}, 0x6352410, 0x32430ab8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc077516390}, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc077516390}, 0xc07aaaa5a0, 0xc07c606ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077516390}, 0x660089, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc077516390}, 0xc05156a500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077516300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077516300}, {0xc07d2508f0, 0x660089}, 0xc07d2508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077516300}, {0xc00102e4e0}, 0xc05156a400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0775162a0}, 0xc05156a400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04e72e600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805765 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016f5da40, {0x63c85b8, 0xc01c569350}, 0xc02c3dc150, 0x1, 0xc005d190a8, 0x519bbc, 0xc04ebe5140, 0xc058ae02a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01c569350}, 0x120, 0xc005d19230, 0x4ebe50e0, 0x0, 0x660069, 0xc04ebe5140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016f5da40, {0x63c85b8, 0xc01c569350}, 0xc04ebe50e0, {{{{0x0, 0x0}, {0xc05529a6c0, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01c569350}, 0x6352410, 0x3a07ebc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016fda500, {0x63c85b8, 0xc01c569350}, {0xc04ebe4ea0, {0x174301103f87867f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016fda500, {0x63c85b8, 0xc01c569350}, 0xc072b91a40, 0xc005d199f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01c569350}, 0x660089, {0xc04ebe4ea0, {0x174301103f87867f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017050000, {0x63c85b8, 0xc01c569350}, 0xc04d55ca00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c5691a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c5691a0}, {0xc005d1b3f8, 0x660089}, 0xc005d1b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c5691a0}, {0xc0015221e0}, 0xc04d55c800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c569110}, 0xc04d55c800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}, 0xc0929cba60, 0xc07995fe78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}, 0x3, 0xc066f93180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005d1b980}}}, {0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}, 0xc0929cba60, 0xc066f93180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01c569110}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c569110}, {0x40, 0xc005d1bb70}, 0xc005d1bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01c569110}, {0x4e9d7a0, 0xc04d55c800}, 0xc0929cba60, 0xc07995fe78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01c569110}, 0xc044c4bc80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc076a75e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc076a75e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219731991 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01816a000, {0x63c85b8, 0xc0c73deb10}, 0xc05a975600, 0xc05a5a1ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01816a000, {0x63c85b8, 0xc0c73deb10}, 0xc05a975600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c228c26, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c228c26, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c73dea80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c73dea80}, {0xc0736407f8, 0x660089}, 0xc073640800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c73dea80}, {0xc00102e4e0}, 0xc05a205500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c73dea20}, 0xc05a205500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0297ea000, {0x63c85b8, 0xc0c73de8d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc053d60000, {0x63c85b8, 0xc0c73de8d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc053d60000, {0x63c8510, 0xc044421b40}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc053d60000, {0x63c8510, 0xc044421b40}, {0x63f7db8, 0x9bf6f48}, 0xc073645088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc053d60000, {0x63c85b8, 0xc01d33b4a0}, {{0x63f7db8, 0x9bf6f48}, {0xc0744b3219, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc053d60000, {0x63c85b8, 0xc01d33b4a0}, {{0x63f7db8, 0x9bf6f48}, {0xc0744b3219, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0744b3219, 0x6}, 0x0, 0x0}, {0x17efdf24, 0xedb7a8350, 0x0}, {0x17efe5fe, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc053d60000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc053d60000, {0x63c8510, 0xc04220f600}, 0xc06da7bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07d3915c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc04220f600}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07d3915c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219798979 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc01eaa7c20) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc073b57550, {0x4d83cc0, 0xc0645b9100}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xcd) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc01eaa79e0, 0xc0869d1048, 0xc0869d1018) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc01eaa79e0, {0x4d83cc0, 0xc0645b9100}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc086fc7200}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc0a085f100}, {0x4d83cc0, 0xc0645b9100}, 0x2, {0xc00c7a1180, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc086fc7200}, {0x4f6406d, 0x7fe457d80878}, {0x4e9d7a0, 0xc0a085f100}, {0x4d83cc0, 0xc0645b9100}, 0x78ba866602eb3bff, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x5ab90962aef6e28d, {0x63c85b8, 0xc086fc7200}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc0a085f100}, {0x4d83cc0, 0xc0645b9100}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc065f8b518, {0x63c85b8, 0xc086fc7200}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc065f8b518}}, {0x63c85b8, 0xc086fc7200}, 0xc0a085f100, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1d, 0x1d, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc04a9c6780, {0x63c85b8, 0xc086fc6f90}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc04a9c6780, {0x63c85b8, 0xc086fc6f90}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc01a21a320, {0x63c85b8, 0xc086fc6f90}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0d09b5500, {0x63c85b8, 0xc086fc6f90}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc05bf954a0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:264 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0286f39c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc086fc6ea0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/is_null_ops.eg.go:226 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc03739bbc0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colexec.(*limitOp).Next(0xc0286f3a00) github.com/cockroachdb/cockroach/pkg/sql/colexec/limit.go:47 +0x3c github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0a73a4c00) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc018a3c2c0, {0x63c8510, 0xc0286f3940}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc04cdbb950, {0x63c8510, 0xc0286f3940}, 0xc041e1ef20) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc0286f35c0}, 0xc09f7cf340, 0xc041e1ef20, 0xc0b21c7e00, 0xc0c32bc700, 0xc0adc49ce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc0286f35c0}, 0xc0adc499f0, 0xc09f7cf340, 0xc052311140, {{0x63ca960, 0xc0cdadc960}, 0x0}, 0xc0c32bc700) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0adc49300, {0x63c8510, 0xc0286f35c0}, 0xc0adc499f0, 0xc0286f35c0, {0x7fe42b790cf0, 0xc052311140}, 0x0, 0xc03739ad78) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0adc49300, {0x63c8510, 0xc0286f35c0}, 0xc0adc499f0, {0x7fe42b790cf0, 0xc052311140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0adc49300, {0x63c85b8, 0xc020dae480}, {{0x63f8d78, 0xc0869223c0}, {0xc05312c720, 0x185}, 0x1, 0x1}, 0xc0437e4120, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0adc49300, {0x63c85b8, 0xc020dae480}, {{0x63f8d78, 0xc0869223c0}, {0xc05312c720, 0x185}, 0x1, 0x1}, 0xc0437e4120, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0adc49300, {0x63c85b8, 0xc020dae480}, {0xc0437e4120, {0xc0554b98e0, 0x1, 0x1}, {0xc05c6ae080, 0x10, 0x10}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc05312c91c, 0x0}, 0x0, {0xc5120c6, 0xedb7a8351, 0x0}, 0x1}, 0xc0adc49300, 0xc0869d7980, 0xc0869d7970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0adc49300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0adc49300, {0x63c8510, 0xc04e2d3080}, 0xc070137dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05b056d00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc04e2d3080}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05b056d00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219799765 [runnable]: sync.runtime_SemacquireMutex(0xc07a9d6690, 0x8f, 0x7fe329bd8780) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc07a9d6538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc07a9d6560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc08c476000, {0xffffffffffffff15, 0xc07a9d66c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc08c476000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc08c476000, {0xc04f04d4a0, 0x32, 0xa0}, {0xc024e43c00, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0931c4398, {0xc04f04d4a0, 0x32, 0xa0}, {0xc024e43c00, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0931c2cc0, {0xc024e43c00, 0x77, 0xa0}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0931c2cc0, {0xc04f04d4a0, 0x77, 0xc024e43c00}, {0xc024e43c00, 0xc063bcf420, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0931c2a00, {0xc024e43c00, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0516e23a8, {{0xc063bcf420, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07abb6a00, {{0xc066ca5860, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ae7e000, {0x63c85b8, 0xc03f32e8a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03f32e8a0}, {0x64a6560, 0xc07abb6a00}, {0xc066ca5860, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03f32e8a0}, {0x6466ed8, 0xc0516e2100}, {0xc066ca5860, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06d54b710}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06432cd00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209ba000, {0x63c85b8, 0xc03f32e8a0}, 0xc06432cd00, 0xc046ded5e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209ba000, {0x63c85b8, 0xc03f32e8a0}, 0xc06432cd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03f32e810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f32e810}, {0xc0ad8388f0, 0x660089}, 0xc0ad8388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03f32e810}, {0xc00102e4e0}, 0xc06432cc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03f32e7b0}, 0xc06432cc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01174b110}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806117 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bbe1b80, {0x63c85b8, 0xc06f484540}, 0xc03cca35f0, 0x1, 0xc001cbb0a8, 0x519bbc, 0xc05a1a6600, 0xc08bf776e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06f484540}, 0x1e0, 0xc001cbb230, 0x5a1a65a0, 0x0, 0x660069, 0xc05a1a6600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bbe1b80, {0x63c85b8, 0xc06f484540}, 0xc05a1a65a0, {{{{0x0, 0x0}, {0xc0477b4120, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06f484540}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bcc4050, {0x63c85b8, 0xc06f484540}, {0xc077cfe480, {0x174301104953add6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bcc4050, {0x63c85b8, 0xc06f484540}, 0xc028f56e60, 0xc001cbb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f484540}, 0x660089, {0xc077cfe480, {0x174301104953add6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bca7c00, {0x63c85b8, 0xc06f484540}, 0xc060bb6d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104953add6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104953add6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f484420}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f484420}, {0xc001cbd3f8, 0x660089}, 0xc001cbd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f484420}, {0xc0015221e0}, 0xc060bb6a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f484330}, 0xc060bb6a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}, 0xc06deb99e0, 0xc086e66900) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}, 0x3, 0xc03c72da40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc001cbd980}}}, {0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}, 0xc06deb99e0, 0xc03c72da40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f484330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f484330}, {0x40, 0xc001cbdb70}, 0xc001cbdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f484330}, {0x4e9d7a0, 0xc060bb6a00}, 0xc06deb99e0, 0xc086e66900) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f484330}, 0xc09382d980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc077cfe360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc077cfe360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812595 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811243 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0210c1c00, {0x63c85b8, 0xc005c3e5d0}, 0xc08d1b4800, 0xc076de5e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c1c00, {0x63c85b8, 0xc005c3e5d0}, 0xc08d1b4800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107426efc8, 0x0, 0x0}, 0xc03cd415d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107426efc8, 0x0, 0x0}, 0xc03cd415d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc005c3e540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc005c3e540}, {0xc0289973f8, 0x660089}, 0xc028997400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc005c3e540}, {0x7fe32e798418}, 0xc08d1b4600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc005c3e4b0}, 0xc08d1b4600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}, 0xc081b09c80, 0xc025b97938) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}, 0x3, 0xc09ac19400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028997980}}}, {0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}, 0xc081b09c80, 0xc09ac19400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc005c3e4b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc005c3e4b0}, {0x40, 0xc028997b70}, 0xc028997ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc005c3e4b0}, {0x4e9d7a0, 0xc08d1b4600}, 0xc081b09c80, 0xc025b97938) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc005c3e4b0}, 0xc050204240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06eb957a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06eb957a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812801 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805104 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc0592a1650}, 0xc05b94e6d8, 0x1, 0x708017db66e6c6, 0x37e11d600, 0xc07124fec0, 0xc03896f5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0592a1650}, 0x51cdcd8, 0xc07a514728, 0x76d8c630, 0x4c166a, 0x18, 0xc07124fec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc0592a1650}, 0xc07124fe60, {{{{0x0, 0x0}, {0xc055a9b200, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0592a1650}, 0x6352410, 0x71c3e988) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc0592a1650}, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc0592a1650}, 0xc04328b360, 0xc076d8cee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0592a1650}, 0x660089, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0592a1650}, 0xc06173ba00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592a15c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592a15c0}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592a15c0}, {0xc00102e4e0}, 0xc06173b900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592a1560}, 0xc06173b900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fcdbe60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799985 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc06fe6e2a0}, 0xc060e50438, 0x1, 0xc07682bf20, 0x519bbc, 0xc040efe1e0, 0xc059ee7aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc06fe6e2a0}, 0xc0, 0xc07682c0a8, 0x40efe180, 0x0, 0x660069, 0xc040efe1e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc06fe6e2a0}, 0xc040efe180, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc06fe6e2a0}, 0x63524a0, 0x63fdde08) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc06fe6e2a0}, {0x0, {0x1743011056ab7792, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc06fe6e2a0}, 0xc02f05c460, 0xc07682c868) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06fe6e2a0}, 0x660089, {0x0, {0x1743011056ab7792, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc06fe6e2a0}, 0xc08d81e500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06fe6e210}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fe6e210}, {0xc07682e270, 0x660089}, 0xc07682e278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06fe6e210}, {0xc07382fda0}, 0xc08d81e400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fe6e1b0}, 0xc08d81e400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x1743011056ab7792, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc046cd7e60}, 0x0, 0xc04eed0680) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc046cd7e60}, 0x66040f) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc019919860, {0x63c85b8, 0xc046cd7e60}, {{0xa8, 0xd0, 0xc8, 0xa9, 0xa1, 0xed, 0x4b, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:922 +0x1b0 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc019919860, {0x63c85b8, 0xc046cd7e60}, 0xc027cf6960, 0xc0402d5680, 0xc03ec93360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:610 +0xd4a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc046cd7e60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc07382fec0}, {{0xc003cf0dc0, 0x24, 0x24}}, 0xc076831380) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc019919860, {0x63c85b8, 0xc07382fec0}, 0xc027cf6960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0301af310, {0x63c85b8, 0xc07382fec0}, {0x0, {0x17430110113816dd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc07382fec0}, 0xc02765a140, 0xc0308909d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07382fec0}, 0xc02765a140, {0x0, {0x17430110113816dd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07382fec0}, 0xc056b35e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110113816dd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110113816dd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x17430110113816dd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07382fe30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07382fe30}, {0xc032b973f8, 0x660089}, 0xc032b97400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07382fe30}, {0x7fe3b7a89428}, 0xc056b35c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07382fda0}, 0xc056b35c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}, 0xc05f699140, 0xc01ced1d88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}, 0x3, 0xc01b91f240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032b97980}}}, {0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}, 0xc05f699140, 0xc01b91f240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07382fda0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07382fda0}, {0x40, 0xc032b97b70}, 0xc032b97ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07382fda0}, {0x4e9d7a0, 0xc056b35c00}, 0xc05f699140, 0xc01ced1d88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07382fda0}, 0xc0386923c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc053ec0d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc053ec0d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810144 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc023bef000, {0x63c85b8, 0xc044117230}, 0xc088312e00, 0xc08f98a960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023bef000, {0x63c85b8, 0xc044117230}, 0xc088312e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c2f405d, 0x0, 0x0}, 0xc04a75ed40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c2f405d, 0x0, 0x0}, 0xc04a75ed40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0441171a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0441171a0}, {0xc0179673f8, 0x660089}, 0xc017967400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0441171a0}, {0x7fe3953db1f8}, 0xc088312c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044117110}, 0xc088312c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}, 0xc07a403240, 0xc03eec3560) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}, 0x3, 0xc07808e300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017967980}}}, {0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}, 0xc07a403240, 0xc07808e300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044117110}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044117110}, {0x40, 0xc017967b70}, 0xc017967ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044117110}, {0x4e9d7a0, 0xc088312c00}, 0xc07a403240, 0xc03eec3560) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044117110}, 0xc074255560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc062b2c480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc062b2c480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812788 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802121 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021ab8000, {0x63c85b8, 0xc0168402d0}, 0xc06d04b000, 0xc0d4392780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021ab8000, {0x63c85b8, 0xc0168402d0}, 0xc06d04b000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020b807ca, 0x0, 0x0}, 0xc06176d0d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020b807ca, 0x0, 0x0}, 0xc06176d0d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc016840180}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc016840180}, {0xc076115918, 0x660089}, 0xc076115920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc016840180}, {0xc00102e4e0}, 0xc06d04af00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc016840120}, 0xc06d04af00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc02f4feb40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc05454eb10}, {0xc00dfd36b0, 0x24}, 0x48301c348, 0xc01728dd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc076c824e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc05454eb10}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806654 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc000c98800, {0x63c85b8, 0xc08cdef890}, 0xc08bbfae00, 0xc049d799a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000c98800, {0x63c85b8, 0xc08cdef890}, 0xc08bbfae00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105089a2d7, 0x0, 0x0}, 0xc0896cb9b0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105089a2d7, 0x0, 0x0}, 0xc0896cb9b0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08cdef800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08cdef800}, {0xc00310a8f0, 0x660089}, 0xc00310a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08cdef800}, {0xc00102e4e0}, 0xc08bbfad00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08cdef7a0}, 0xc08bbfad00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc08d3dd5f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803927 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fc5ca00, {0x63c85b8, 0xc04d8a14a0}, 0xc02be667c8, 0x1, 0xc0774ef0a8, 0x519bbc, 0xc04dd675c0, 0xc04e226a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04d8a14a0}, 0x120, 0xc0774ef230, 0x4dd67560, 0x0, 0x660069, 0xc04dd675c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fc5ca00, {0x63c85b8, 0xc04d8a14a0}, 0xc04dd67560, {{{{0x0, 0x0}, {0xc0731b4c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d8a14a0}, 0x6352410, 0x14c75d98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fc6a3c0, {0x63c85b8, 0xc04d8a14a0}, {0xc04dd67440, {0x174301103074bebc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fc6a3c0, {0x63c85b8, 0xc04d8a14a0}, 0xc058290e60, 0xc0774ef9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d8a14a0}, 0x660089, {0xc04dd67440, {0x174301103074bebc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fc82000, {0x63c85b8, 0xc04d8a14a0}, 0xc070567d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103074bebc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103074bebc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d8a1380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d8a1380}, {0xc0774f13f8, 0x660089}, 0xc0774f1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d8a1380}, {0xc03e522190}, 0xc070567b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d8a12f0}, 0xc070567b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}, 0xc085621040, 0xc0b43ad458) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}, 0x3, 0xc06d189780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0774f1980}}}, {0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}, 0xc085621040, 0xc06d189780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d8a12f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d8a12f0}, {0x40, 0xc0774f1b70}, 0xc0774f1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d8a12f0}, {0x4e9d7a0, 0xc070567b00}, 0xc085621040, 0xc0b43ad458) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d8a12f0}, 0xc05b148420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04dd67320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04dd67320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799949 [runnable]: sync.runtime_SemacquireMutex(0xc091b2e690, 0x8f, 0x7fe1abdb4980) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc091b2e538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc091b2e560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc045d66000, {0xffffffffffffff15, 0xc091b2e6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc045d66000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc045d66000, {0xc0830f7b90, 0x32, 0x6e}, {0xc0e4232540, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07a091998, {0xc0830f7b90, 0x32, 0x6e}, {0xc0e4232540, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07a0902c0, {0xc0e4232540, 0x32, 0x6e}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07a0902c0, {0xc0830f7b90, 0x32, 0xc0e4232540}, {0xc0e4232540, 0xc040427320, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07a090000, {0xc0e4232540, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc052de9328, {{0xc040427320, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc086d70f00, {{0xc0c7380300, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04b392800, {0x63c85b8, 0xc01fd01d10}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01fd01d10}, {0x64a6560, 0xc086d70f00}, {0xc0c7380300, 0x20907, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01fd01d10}, {0x6466ed8, 0xc052de9080}, {0xc0c7380300, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc046ebe468}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03bfc9a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209f0400, {0x63c85b8, 0xc01fd01d10}, 0xc03bfc9a00, 0xc054757ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209f0400, {0x63c85b8, 0xc01fd01d10}, 0xc03bfc9a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fd01c80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fd01c80}, {0xc0058548f0, 0x660089}, 0xc0058548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fd01c80}, {0xc00102e4e0}, 0xc03bfc9900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fd01c20}, 0xc03bfc9900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0241225a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804490 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc047449200}, 0xc00065bbd8, 0x1, 0xc07217b0a8, 0x519bbc, 0xc06035e180, 0xc051c8f6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc047449200}, 0x120, 0xc07217b230, 0x6035e120, 0x0, 0x660069, 0xc06035e180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc047449200}, 0xc06035e120, {{{{0x0, 0x0}, {0xc05c3e0000, 0xf0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047449200}, 0x6352410, 0x45bf5718) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc047449200}, {0xc06035e000, {0x17430110321ddba8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc047449200}, 0xc02d48c960, 0xc07217b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047449200}, 0x660089, {0xc06035e000, {0x17430110321ddba8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc047449200}, 0xc04a6e0300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110321ddba8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110321ddba8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047449170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047449170}, {0xc07217d3f8, 0x660089}, 0xc07217d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047449170}, {0x7fe3c2fd68a8}, 0xc04a6e0000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0474490e0}, 0xc04a6e0000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}, 0xc08cfa7440, 0xc02a062810) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}, 0x3, 0xc06ba4c800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07217d980}}}, {0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}, 0xc08cfa7440, 0xc06ba4c800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0474490e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0474490e0}, {0x40, 0xc07217db70}, 0xc07217dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0474490e0}, {0x4e9d7a0, 0xc04a6e0000}, 0xc08cfa7440, 0xc02a062810) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0474490e0}, 0xc046d20c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05f3b5e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05f3b5e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802526 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030040800, {0x63c85b8, 0xc048d5d1d0}, 0xc06f446c00, 0xc077741900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030040800, {0x63c85b8, 0xc048d5d1d0}, 0xc06f446c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffc530edc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffc530edc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048d5d140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048d5d140}, {0xc0554233f8, 0x660089}, 0xc055423400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048d5d140}, {0x7fe3ba7da478}, 0xc06f446a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048d5d080}, 0xc06f446a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}, 0xc05f207a00, 0xc04ccd5c20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}, 0x3, 0xc059076440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055423980}}}, {0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}, 0xc05f207a00, 0xc059076440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048d5d080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048d5d080}, {0x40, 0xc055423b70}, 0xc055423ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048d5d080}, {0x4e9d7a0, 0xc06f446a00}, 0xc05f207a00, 0xc04ccd5c20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048d5d080}, 0xc01bcad8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0473738c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0473738c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809197 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc01e0a1290}, 0xc0439193e0, 0x1, 0xc010b390a8, 0x519bbc, 0xc052b79380, 0xc05f77e720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01e0a1290}, 0x120, 0xc010b39230, 0x52b79320, 0x0, 0x660069, 0xc052b79380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc01e0a1290}, 0xc052b79320, {{{{0x0, 0x0}, {0xc08fa2f440, 0xa5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01e0a1290}, 0x6352410, 0x450cb0c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc01e0a1290}, {0xc052b79200, {0x1743011063874365, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc01e0a1290}, 0xc0c70020a0, 0xc010b399f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01e0a1290}, 0x660089, {0xc052b79200, {0x1743011063874365, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01e0a1290}, 0xc03de42300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063874365, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063874365, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e0a11d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e0a11d0}, {0xc010b3b3f8, 0x660089}, 0xc010b3b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e0a11d0}, {0x6e5f6c616e6f6974}, 0xc03de42000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e0a10b0}, 0xc03de42000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}, 0xc079f16f60, 0xc056a758f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}, 0x3, 0xc085a33ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010b3b980}}}, {0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}, 0xc079f16f60, 0xc085a33ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01e0a10b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e0a10b0}, {0x40, 0xc010b3bb70}, 0xc010b3bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01e0a10b0}, {0x4e9d7a0, 0xc03de42000}, 0xc079f16f60, 0xc056a758f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01e0a10b0}, 0xc06754a240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052b790e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052b790e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800908 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030309c00, {0x63c85b8, 0xc057c51a10}, 0xc00a8e8500, 0xc0b7e76b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030309c00, {0x63c85b8, 0xc057c51a10}, 0xc00a8e8500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101936f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101936f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057c51950}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057c51950}, {0xc0471013f8, 0x660089}, 0xc047101400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057c51950}, {0x0}, 0xc00a8e8300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057c518c0}, 0xc00a8e8300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}, 0xc06b8dd560, 0xc070f85758) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}, 0x3, 0xc044546ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc047101980}}}, {0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}, 0xc06b8dd560, 0xc044546ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc057c518c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057c518c0}, {0x40, 0xc047101b70}, 0xc047101ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc057c518c0}, {0x4e9d7a0, 0xc00a8e8300}, 0xc06b8dd560, 0xc070f85758) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc057c518c0}, 0xc06b8886c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc064a87c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc064a87c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811154 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021deedc0, {0x63c85b8, 0xc06742af30}, 0xc0bb3a8978, 0x1, 0xc013b890a8, 0x519bbc, 0xc033cf5920, 0xc0f4670a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06742af30}, 0x120, 0xc013b89230, 0x33cf58c0, 0x0, 0x660069, 0xc033cf5920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021deedc0, {0x63c85b8, 0xc06742af30}, 0xc033cf58c0, {{{{0x0, 0x0}, {0xc086d97440, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06742af30}, 0x6352410, 0x7818f960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3d900, {0x63c85b8, 0xc06742af30}, {0xc033cf57a0, {0x1743011063a42e49, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3d900, {0x63c85b8, 0xc06742af30}, 0xc05292f220, 0xc013b899f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06742af30}, 0x660089, {0xc033cf57a0, {0x1743011063a42e49, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021e97c00, {0x63c85b8, 0xc06742af30}, 0xc04919cc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063a42e49, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063a42e49, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06742aea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06742aea0}, {0xc013b8b3f8, 0x660089}, 0xc013b8b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06742aea0}, {0x40df00bf96283369}, 0xc04919ca00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06742ae10}, 0xc04919ca00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}, 0xc02581cf80, 0xc012010630) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}, 0x3, 0xc04559b480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc013b8b980}}}, {0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}, 0xc02581cf80, 0xc04559b480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06742ae10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06742ae10}, {0x40, 0xc013b8bb70}, 0xc013b8bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06742ae10}, {0x4e9d7a0, 0xc04919ca00}, 0xc02581cf80, 0xc012010630) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06742ae10}, 0xc0145c35c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc033cf5560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc033cf5560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799767 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc094fd2548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc094fd25b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0893d20f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0544fb500, {0xc083886150, 0x2b, 0x70}, {0xc078a06be0, 0x2b, 0x94}, 0x1, 0x20) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0893d1f88, {0xc083886150, 0x1, 0x94}, {0xc078a06be0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0893d1f88, {0xc083886150, 0x2b, 0x70}, {0xc078a06be0, 0x2b, 0x94}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0893d16c0, {0xc078a06be0, 0x94, 0x70}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0893d16c0, {0xc083886150, 0x2b, 0xc078a06be0}, {0xc078a06be0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0893d1400, {0xc078a06be0, 0x2b, 0x94}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04c100688, {{0xc079ce19b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0556bec00, {{0xc079ce19b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0cd8ea800, {0x63c85b8, 0xc017fd1320}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc017fd1320}, {0x64a6560, 0xc0556bec00}, {0xc079ce19b0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc017fd1320}, {0x6466ed8, 0xc04c100680}, {0xc079ce19b0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06637f8c0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0563ca000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209bb800, {0x63c85b8, 0xc017fd1320}, 0xc0563ca000, 0xc071cbc0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209bb800, {0x63c85b8, 0xc017fd1320}, 0xc0563ca000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc017fd1290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc017fd1290}, {0xc0076c88f0, 0x660089}, 0xc0076c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc017fd1290}, {0xc00102e4e0}, 0xc00b5fdb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc017fd1200}, 0xc00b5fdb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01174b1d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809299 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02efcb000, {0x63c85b8, 0xc053ed8f90}, 0xc04b482500, 0xc057f90a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02efcb000, {0x63c85b8, 0xc053ed8f90}, 0xc04b482500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063da7d74, 0x0, 0x0}, 0xc04d582120, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063da7d74, 0x0, 0x0}, 0xc04d582120, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053ed8f00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053ed8f00}, {0xc013be53f8, 0x660089}, 0xc013be5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053ed8f00}, {0x7fe351ee36c8}, 0xc04b482300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053ed8e70}, 0xc04b482300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}, 0xc0761eac40, 0xc049a02d38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}, 0x3, 0xc011f1a600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc013be5980}}}, {0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}, 0xc0761eac40, 0xc011f1a600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc053ed8e70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053ed8e70}, {0x40, 0xc013be5b70}, 0xc013be5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc053ed8e70}, {0x4e9d7a0, 0xc04b482300}, 0xc0761eac40, 0xc049a02d38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc053ed8e70}, 0xc02da79980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0c131aa20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0c131aa20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801822 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc048710ae0}, 0xc02ecc9ad0, 0x1, 0xc055ebd0a8, 0x519bbc, 0xc016589aa0, 0xc033c21e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc048710ae0}, 0x1e0, 0xc055ebd230, 0x16589a40, 0x0, 0x660069, 0xc016589aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc048710ae0}, 0xc016589a40, {{{{0x0, 0x0}, {0xc036538000, 0x30}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc048710ae0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc048710ae0}, {0xc041514d80, {0x174301101d2e34ec, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc048710ae0}, 0xc04a99bae0, 0xc055ebd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc048710ae0}, 0x660089, {0xc041514d80, {0x174301101d2e34ec, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc048710ae0}, 0xc049b18700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d2e34ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d2e34ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048710a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048710a20}, {0xc055ebf3f8, 0x660089}, 0xc055ebf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048710a20}, {0x7fe3c2e886b8}, 0xc04dbb3f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048710990}, 0xc04dbb3f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}, 0xc021fd6b20, 0xc02257d608) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}, 0x3, 0xc036f8df00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055ebf980}}}, {0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}, 0xc021fd6b20, 0xc036f8df00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048710990}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048710990}, {0x40, 0xc055ebfb70}, 0xc055ebfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048710990}, {0x4e9d7a0, 0xc04dbb3f00}, 0xc021fd6b20, 0xc02257d608) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048710990}, 0xc03dfdef60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc041514c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc041514c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805075 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc06c2f8570}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219775813 [runnable]: internal/poll.runtime_pollWait(0x7fdbaffb3290, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0607fbe80, 0xc02066a800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0607fbe80, {0xc02066a800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0607fbe80, {0xc02066a800, 0x735, 0xc046779180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc065bed360, {0xc02066a800, 0x7eda79, 0xc02066a805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0ccee8360, {0xc02066a800, 0x18, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0342fcf18, {0xc02066a800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc090aa3778, {0x631b900, 0xc0342fcf18}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc090aa3500, {0x7fe42ab59310, 0xc0ccee8360}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc090aa3500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc090aa3500, {0xc053cdf000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06d598360, {0xc053cdf000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0965b6b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0965b6b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0965b70e0, {0x634aac0, 0xc0965b6b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0965b6a80, 0xc01ee43af0, 0xc08684b42b, 0xc08684b42d, {0x635bec0, 0xc052a6ad80}, 0xc000d6ae00, 0xc0435eea98, 0xc08684b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0965b6a80, {0x63c85b8, 0xc06f6c9a40}, 0xc01ee43ae0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc02999e020, 0x8}}, 0x0, 0xc06f6c99e0, 0xc06f6c9a10, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06f6c9a40}, {0x643d810, 0xc0ccee8360}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04d363b60}, {0x643d810, 0xc0ccee8360}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04d363b60}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812363 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc06bd1a750, {0x63c85b8, 0xc06c1f4a80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc06c1f4a80}, {0x4f67ad2, 0x10beec0}, 0xc002c9efb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219800853 [runnable]: sync.runtime_SemacquireMutex(0xc076e36690, 0x8f, 0x7fd4552aa000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc076e36538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc076e36560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc028448d80, {0xffffffffffffff15, 0xc076e366c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc028448d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc028448d80, {0xc006bec5a0, 0x32, 0x8e}, {0xc06211f560, 0x32, 0x5d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc076dda898, {0xc006bec5a0, 0x32, 0x8e}, {0xc06211f560, 0x32, 0x5d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc076dd91c0, {0xc06211f560, 0x5d, 0x8e}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc076dd91c0, {0xc006bec5a0, 0x5d, 0xc06211f560}, {0xc06211f560, 0xc060ce4ab0, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc076dd8f00, {0xc06211f560, 0x32, 0x5d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc07c03d9a8, {{0xc060ce4ab0, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc097299d00, {{0xc070b16090, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc025f3d400, {0x63c85b8, 0xc060d2a570}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc060d2a570}, {0x64a6560, 0xc097299d00}, {0xc070b16090, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc060d2a570}, {0x6466ed8, 0xc07c03d700}, {0xc070b16090, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc014ba8a08}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03a7c7a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c00000, {0x63c85b8, 0xc060d2a570}, 0xc03a7c7a00, 0xc07a3a7040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c00000, {0x63c85b8, 0xc060d2a570}, 0xc03a7c7a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060d2a4e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060d2a4e0}, {0xc07d2528f0, 0x660089}, 0xc07d2528f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060d2a4e0}, {0xc00102e4e0}, 0xc03a7c7900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc060d2a480}, 0xc03a7c7900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a45a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812625 [runnable]: github.com/cockroachdb/pebble/internal/manual._Cfunc_calloc(0x7fd3, 0x1) github.com/cockroachdb/pebble/internal/manual/_cgo_gotypes.go:46 +0x4d github.com/cockroachdb/pebble/internal/manual.New(0x7fd3) /home/roach/.cache/bazel/_bazel_roach/c5a4e7d36696d9cd970af2045211a7df/sandbox/processwrapper-sandbox/3244/execroot/cockroach/external/com_github_cockroachdb_pebble/internal/manual/manual.go:40 +0x34 github.com/cockroachdb/pebble/internal/cache.newValue(0xc0146f10f8) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/value_normal.go:39 +0x26 github.com/cockroachdb/pebble/internal/cache.(*Cache).Alloc(...) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:788 github.com/cockroachdb/pebble/sstable.decompressBlock(0xc015e588a0, 0x1, {0x7fd7425f86a0, 0x6803, 0x6808}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/compression.go:69 +0xd4 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc016113080, {0x373ad93, 0x6803}, 0x0, 0xc04e70aff8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2353 +0x499 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04e70ad80, {0xffffffffffffff15, 0xc043b310c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc04e70ad80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc04e70ad80, {0xc083762460, 0x28, 0x67}, 0x13, 0xef) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04e70ad80, {0xc01bdd7f00, 0x28, 0x67}, {0xc083762460, 0x0, 0xc04e70ad80}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04e70ad80, {0xc01bdd7f00, 0x28, 0x7f}, {0xc083762460, 0x28, 0x67}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc078c97298, {0xc01bdd7f00, 0x28, 0x7f}, {0xc083762460, 0x28, 0x67}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc078c95bc0, {0xc083762460, 0x0, 0xc043b314e0}, 0x121ea30, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc078c95bc0, {0xc01bdd7f00, 0x28, 0x67}, {0xc083762460, 0x27, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc078c95900, {0xc083762460, 0x28, 0x67}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc052deb708, {{0xc03d575920, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc056d7d400, {{0xc03d575920, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc051ebe000, {0x63c85b8, 0xc02b87b410}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02b87b410}, {0x64a6560, 0xc056d7d400}, {0xc03d575920, 0x0, 0x30}, {0x174301109996abe8, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02b87b410}, {0x6466ed8, 0xc052deb700}, {0xc03d575920, 0x27, 0x30}, {0x174301109996abe8, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0234ea498}, {{0x174301109996abe8, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109996abe8, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc07314fd00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b76410e8, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d7c00, {0x63c85b8, 0xc02b87b410}, 0xc07314fd00, 0xc066551180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d7c00, {0x63c85b8, 0xc02b87b410}, 0xc07314fd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109996abe8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109996abe8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b87b080}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b87b080}, {0xc043b373f8, 0x660089}, 0xc043b37400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b87b080}, {0x0}, 0xc07314fb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02b87ae40}, 0xc07314fb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}, 0xc019cca1a0, 0xc060f8e900) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}, 0x3, 0xc015c8e740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043b37980}}}, {0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}, 0xc019cca1a0, 0xc015c8e740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02b87ae40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b87ae40}, {0x40, 0xc043b37b70}, 0xc043b37ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02b87ae40}, {0x4e9d7a0, 0xc07314fb00}, 0xc019cca1a0, 0xc060f8e900) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02b87ae40}, 0xc08bd8bbc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800623 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020f4a000, {0x63c85b8, 0xc06d857530}, 0xc04dfe1400, 0xc06149ad20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4a000, {0x63c85b8, 0xc06d857530}, 0xc04dfe1400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101644929a, 0x0, 0x0}, 0xc063dceef0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101644929a, 0x0, 0x0}, 0xc063dceef0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06d8574a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d8574a0}, {0xc0766d3918, 0x660089}, 0xc0766d3920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06d8574a0}, {0xc00102e4e0}, 0xc04dfe1300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06d857440}, 0xc04dfe1300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc035ec9320}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06d8573b0}, {0xc00dfd36b0, 0x24}, 0x46c753f38, 0xc0117d6d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc07addff80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc06d8573b0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800527 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01ed76400, {0x63c85b8, 0xc0768a6cc0}, 0xc034450a00, 0xc05292ebe0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ed76400, {0x63c85b8, 0xc0768a6cc0}, 0xc034450a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011016208a73, 0x0, 0x0}, 0xc0080735e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011016208a73, 0x0, 0x0}, 0xc0080735e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0768a6b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0768a6b10}, {0xc04804f3f8, 0x660089}, 0xc04804f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0768a6b10}, {0x7461636f6c22202c}, 0xc034450800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0768a6a80}, 0xc034450800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}, 0xc059441d40, 0xc0419d5500) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}, 0x3, 0xc01f39f180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04804f980}}}, {0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}, 0xc059441d40, 0xc01f39f180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0768a6a80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0768a6a80}, {0x40, 0xc04804fb70}, 0xc04804fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0768a6a80}, {0x4e9d7a0, 0xc034450800}, 0xc059441d40, 0xc0419d5500) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0768a6a80}, 0xc042502d80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0441b2120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0441b2120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807335 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc05289a180}, 0xc030e76708, 0x1, 0xc0051e70a8, 0x519bbc, 0xc03b7e7040, 0xc06c2329c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc074e8d0e0, {0x63c85b8, 0xc05289a180}, 0xc074e8d130, 0xc0051e71a8, 0x41, 0x60, 0xffffffffffffffff, 0xc03b7e7040, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc05289a180}, 0xc03b7e6e00, {{{{0x0, 0x0}, {0xc061f60fc0, 0x6}}, {{0xc0520cdc20, 0xb}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05289a180}, 0x63521a0, 0x3b956750) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc05289a180}, {0xc0791cdb00, {0x1743010ffdc03949, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc05289a180}, 0xc0312f2820, 0xc0051e79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05289a180}, 0x660089, {0xc0791cdb00, {0x1743010ffdc03949, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc05289a180}, 0xc08e248700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffdc03949, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffdc03949, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05289a0f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05289a0f0}, {0xc0051e93f8, 0x660089}, 0xc0051e9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05289a0f0}, {0x0}, 0xc08e248500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc023875f50}, 0xc08e248500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}, 0xc0412c9a80, 0xc042adc330) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}, 0x3, 0xc078a45f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0051e9980}}}, {0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}, 0xc0412c9a80, 0xc078a45f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc023875f50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023875f50}, {0x40, 0xc0051e9b70}, 0xc0051e9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc023875f50}, {0x4e9d7a0, 0xc08e248500}, 0xc0412c9a80, 0xc042adc330) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc023875f50}, 0xc06ea339e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07c42e480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07c42e480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808563 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc01d030f60}, 0xc034162390, 0x1, 0xc0080b1ff0, 0x519bbc, 0xc01a01d6e0, 0xc037ced8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc01d030f60}, 0x1e0, 0xc0080b2178, 0x1a01d680, 0x0, 0x660069, 0xc01a01d6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc01d030f60}, 0xc01a01d680, {{{{0x0, 0x0}, {0xc04c14b680, 0x51}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc01d030f60}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc01d030f60}, {0xc061011200, {0x174301105dec2487, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc01d030f60}, 0xc02c88c820, 0xc07d138938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01d030f60}, 0x660089, {0xc061011200, {0x174301105dec2487, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc01d030f60}, 0xc045537500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105dec2487, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105dec2487, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01d030c60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d030c60}, {0xc07d13a340, 0x660089}, 0xc07d13a348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01d030c60}, {0xc00102e4e0}, 0xc045537400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01d030c00}, 0xc045537400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc01d030390}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812449 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc073ab3080}, 0xc02b06aa50, 0x1, 0x7080183b7bc1bd, 0x37e11d600, 0xc00de16780, 0xc040eb9980) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0742b8000, {0x63c85b8, 0xc073ab3080}, 0xc0344d6000, 0xc042cdd1a8, 0xde16780, 0x0, 0x3, 0xc00de16780, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc073ab3080}, 0xc00de166c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc053bd9680, 0xb6}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073ab3080}, 0x6352260, 0x36ce8340) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc073ab3080}, {0xc00de16480, {0x174301104bee1d80, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc073ab3080}, 0xc02f9cec80, 0xc042cdd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073ab3080}, 0x660089, {0xc00de16480, {0x174301104bee1d80, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc073ab3080}, 0xc065f95000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104bee1d80, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104bee1d80, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073ab2ff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073ab2ff0}, {0xc042cdf3f8, 0x660089}, 0xc042cdf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073ab2ff0}, {0x7fe429307910}, 0xc065f94e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073ab2f60}, 0xc065f94e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}, 0xc06d8c03a0, 0xc02b06aa20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}, 0x3, 0xc04df40cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042cdf980}}}, {0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}, 0xc06d8c03a0, 0xc04df40cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073ab2f60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073ab2f60}, {0x40, 0xc042cdfb70}, 0xc042cdfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073ab2f60}, {0x4e9d7a0, 0xc065f94e00}, 0xc06d8c03a0, 0xc02b06aa20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073ab2f60}, 0xc02423d860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e9560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e9560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812597 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812188 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bcd0140, {0x63c85b8, 0xc02728a210}, 0xc01e8846d8, 0x1, 0xc03995d0a8, 0x519bbc, 0xc048bfca80, 0xc062bf8ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02728a210}, 0x120, 0xc03995d230, 0x48bfca20, 0x0, 0x660069, 0xc048bfca80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bcd0140, {0x63c85b8, 0xc02728a210}, 0xc048bfca20, {{{{0x0, 0x0}, {0xc077de10e0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02728a210}, 0x6352410, 0x33a92980) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bcc42d0, {0x63c85b8, 0xc02728a210}, {0xc048bfc900, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bcc42d0, {0x63c85b8, 0xc02728a210}, 0xc047cd5540, 0xc03995d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02728a210}, 0x660089, {0xc048bfc900, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bcd2c00, {0x63c85b8, 0xc02728a210}, 0xc02b640600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02728a180}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02728a180}, {0xc03995f3f8, 0x660089}, 0xc03995f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02728a180}, {0x752b5471556a7770}, 0xc02b640400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02728a0f0}, 0xc02b640400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}, 0xc05bd8b4c0, 0xc01e885710) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}, 0x3, 0xc029c25440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03995f980}}}, {0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}, 0xc05bd8b4c0, 0xc029c25440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02728a0f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02728a0f0}, {0x40, 0xc03995fb70}, 0xc03995fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02728a0f0}, {0x4e9d7a0, 0xc02b640400}, 0xc05bd8b4c0, 0xc01e885710) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02728a0f0}, 0xc0270eb860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f31320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f31320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806269 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0871a3920}, 0xc08a7eb0c8, 0x1, 0x0, 0x1, 0xc01e54d6e0, 0xc055da6cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0871a3920}, 0x120, 0xc00b4ec728, 0x1e54d680, 0x0, 0x660069, 0xc01e54d6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0871a3920}, 0xc01e54d680, {{{{0x0, 0x0}, {0xc03627c000, 0x104}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0871a3920}, 0x6352410, 0x45bf4e08) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0871a3920}, {0xc04c705d40, {0x174301104e7003bc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0871a3920}, 0xc05292eaa0, 0xc07b0d4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0871a3920}, 0x660089, {0xc04c705d40, {0x174301104e7003bc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0871a3920}, 0xc0612c7900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e7003bc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e7003bc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0871a3890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0871a3890}, {0xc0036f68f0, 0x660089}, 0xc0036f68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0871a3890}, {0xc00102e4e0}, 0xc0612c7800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0871a3830}, 0xc0612c7800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0871a34a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812051 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc05f20b590}, 0xc04479ba88, 0x1, 0x0, 0x1, 0xc055f8a2a0, 0xc058064ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05f20b590}, 0x120, 0xc029e40728, 0x55f8a240, 0x0, 0x660069, 0xc055f8a2a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc05f20b590}, 0xc055f8a240, {{{{0x0, 0x0}, {0xc035cd7200, 0x33}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05f20b590}, 0x6352410, 0x204d2468) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc05f20b590}, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc05f20b590}, 0xc01c9beb40, 0xc07d278ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05f20b590}, 0x660089, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc05f20b590}, 0xc010bc5c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05f20b500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f20b500}, {0xc0110d68f0, 0x660089}, 0xc0110d68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05f20b500}, {0xc00102e4e0}, 0xc010bc5b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05f20b470}, 0xc010bc5b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0725422a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806663 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc053bf5250, {0x63c85b8, 0xc08cdef8c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc08cdef8c0}, {0x4f67ad2, 0x10beec0}, 0xc032b1cfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219802172 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc0314f5f50}, 0xc03a2897a0, 0x1, 0xc053aa90a8, 0x519bbc, 0xc03da80840, 0xc0457474a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0314f5f50}, 0x120, 0xc053aa9230, 0x3da807e0, 0x0, 0x660069, 0xc03da80840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc0314f5f50}, 0xc03da807e0, {{{{0x0, 0x0}, {0xc048879d40, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0314f5f50}, 0x6352410, 0x41bc4028) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc0314f5f50}, {0xc03da806c0, {0x1743011021e23374, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc0314f5f50}, 0xc04646b0e0, 0xc053aa99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0314f5f50}, 0x660089, {0xc03da806c0, {0x1743011021e23374, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc0314f5f50}, 0xc0ab104a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011021e23374, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011021e23374, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0314f5cb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0314f5cb0}, {0xc053aab3f8, 0x660089}, 0xc053aab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0314f5cb0}, {0xc02e8fdd58}, 0xc0ab104800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0314f5a40}, 0xc0ab104800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}, 0xc00a33cc40, 0xc0a29e6cf0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}, 0x3, 0xc043d2bc80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc053aab980}}}, {0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}, 0xc00a33cc40, 0xc043d2bc80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0314f5a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0314f5a40}, {0x40, 0xc053aabb70}, 0xc053aabab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0314f5a40}, {0x4e9d7a0, 0xc0ab104800}, 0xc00a33cc40, 0xc0a29e6cf0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0314f5a40}, 0xc028beff20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03da805a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03da805a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810733 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018112640, {0x63c85b8, 0xc0c73dec90}, 0xc0181c2db0, 0x1, 0x0, 0x1, 0xc0618642a0, 0xc056dde720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0c73dec90}, 0x120, 0xc026bac728, 0x61864240, 0x0, 0x660069, 0xc0618642a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018112640, {0x63c85b8, 0xc0c73dec90}, 0xc061864240, {{{{0x0, 0x0}, {0xc0639cf320, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0c73dec90}, 0x6352410, 0x20fd3880) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc017e6f4f0, {0x63c85b8, 0xc0c73dec90}, {0xc06526cfc0, {0x174301106c228c26, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc017e6f4f0, {0x63c85b8, 0xc0c73dec90}, 0xc03baa1540, 0xc07dd20ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0c73dec90}, 0x660089, {0xc06526cfc0, {0x174301106c228c26, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01816a000, {0x63c85b8, 0xc0c73dec90}, 0xc036478100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c228c26, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c228c26, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c73dec00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c73dec00}, {0xc07d2508f0, 0x660089}, 0xc07d2508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c73dec00}, {0x24}, 0xc036478000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c73deba0}, 0xc036478000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06e8d6480}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 213794559 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bf9fd38, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc033aa2c80, 0xc099ab9000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc033aa2c80, {0xc099ab9000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc033aa2c80, {0xc099ab9000, 0x5, 0xc08e154bc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06af87a00, {0xc099ab9000, 0x80, 0x7f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04d0f0e40, {0xc099ab9000, 0x4bf403, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2690f00, {0xc099ab9000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc099711e78, {0x631b900, 0xc0b2690f00}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc099711c00, {0x7fe42ab59310, 0xc04d0f0e40}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc099711c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc099711c00, {0xc09dc53000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0496f8810, {0xc09dc53000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc09dc5e088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc09dc5e088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc09dc5e660, {0x634aac0, 0xc09dc5e088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc09dc5e000, 0xc057e185f0, 0xc08e15542b, 0xc08e15542d, {0x635bec0, 0xc03b36be60}, 0xc000d6ae00, 0xc08d68b178, 0xc08e155429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc09dc5e000, {0x63c85b8, 0xc0705af110}, 0xc057e185e0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09dc4f026, 0x8}}, 0x0, 0xc0705af0b0, 0xc0705af0e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0705af110}, {0x643d810, 0xc04d0f0e40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc066d30ab0}, {0x643d810, 0xc04d0f0e40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc066d30ab0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807919 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a5e3540, {0x63c85b8, 0xc04e72ec30}, 0xc0722816c8, 0x1, 0x0, 0x1, 0xc08ceb2ba0, 0xc05be52240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04e72ec30}, 0x1e0, 0xc00e74a728, 0x8ceb2b40, 0x0, 0x660069, 0xc08ceb2ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a5e3540, {0x63c85b8, 0xc04e72ec30}, 0xc08ceb2b40, {{{{0x0, 0x0}, {0xc046c1db00, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x20300e, {0x63c85b8, 0xc04e72ec30}, 0xc0ee6385c8, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a5d6c30, {0x63c85b8, 0xc04e72ec30}, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a5d6c30, {0x63c85b8, 0xc04e72ec30}, 0xc032769ae0, 0xc07c606ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e72ec30}, 0x660089, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a61d400, {0x63c85b8, 0xc04e72ec30}, 0xc09ccc7900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e72eba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e72eba0}, {0xc07d2508f0, 0x660089}, 0xc07d2508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e72eba0}, {0xc00102e4e0}, 0xc09ccc7800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e72eb40}, 0xc09ccc7800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04e72e240}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810251 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024a89400, {0x63c85b8, 0xc079483590}, 0xc08467fc00, 0xc057f208c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024a89400, {0x63c85b8, 0xc079483590}, 0xc08467fc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106ca6728b, 0x0, 0x0}, 0xc04ac86a50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106ca6728b, 0x0, 0x0}, 0xc04ac86a50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079483500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079483500}, {0xc025be73f8, 0x660089}, 0xc025be7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079483500}, {0x7fe3f73d2378}, 0xc08467fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079483470}, 0xc08467fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}, 0xc01dcf2de0, 0xc0462f79e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}, 0x3, 0xc07ba16000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc025be7980}}}, {0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}, 0xc01dcf2de0, 0xc07ba16000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079483470}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079483470}, {0x40, 0xc025be7b70}, 0xc025be7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079483470}, {0x4e9d7a0, 0xc08467fa00}, 0xc01dcf2de0, 0xc0462f79e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079483470}, 0xc01ceb2960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0616e38c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0616e38c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808469 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01352cf00, {0x63c85b8, 0xc067e0f800}, 0xc0285b2330, 0x1, 0xc011bb7ff0, 0x519bbc, 0xc05ab87c80, 0xc052135ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc067e0f800}, 0x120, 0xc011bb8178, 0x5ab87c20, 0x0, 0x660069, 0xc05ab87c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01352cf00, {0x63c85b8, 0xc067e0f800}, 0xc05ab87c20, {{{{0x0, 0x0}, {0xc0851797a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc067e0f800}, 0x6352410, 0x53d3d368) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc008377360, {0x63c85b8, 0xc067e0f800}, {0xc05ab87b00, {0x174301105bff062f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc008377360, {0x63c85b8, 0xc067e0f800}, 0xc051998820, 0xc07ce42938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067e0f800}, 0x660089, {0xc05ab87b00, {0x174301105bff062f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000dd1400, {0x63c85b8, 0xc067e0f800}, 0xc015105800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067e0f770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067e0f770}, {0xc07ce44340, 0x660089}, 0xc07ce44348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067e0f770}, {0xc00102e4e0}, 0xc015105700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067e0f710}, 0xc015105700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc04aa59620}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802213 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc027d94bd0}, 0xc075e1a738, 0x1, 0xc056caf0a8, 0x519bbc, 0xc0d995f020, 0xc06794dda0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc027d94bd0}, 0x120, 0xc056caf230, 0xd995efc0, 0x0, 0x660069, 0xc0d995f020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc027d94bd0}, 0xc0d995efc0, {{{{0x0, 0x0}, {0xc031e78fc0, 0xa5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc027d94bd0}, 0x6352410, 0x55ace778) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc027d94bd0}, {0xc0d995eea0, {0x1743011020f6199f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc027d94bd0}, 0xc075ed5860, 0xc056caf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc027d94bd0}, 0x660089, {0xc0d995eea0, {0x1743011020f6199f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc027d94bd0}, 0xc035820400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020f6199f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020f6199f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027d94b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027d94b10}, {0xc056cb13f8, 0x660089}, 0xc056cb1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027d94b10}, {0xc05aec8be0}, 0xc035820200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027d94a80}, 0xc035820200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}, 0xc0b9208880, 0xc052a94558) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}, 0x3, 0xc01f6cc6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc056cb1980}}}, {0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}, 0xc0b9208880, 0xc01f6cc6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc027d94a80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027d94a80}, {0x40, 0xc056cb1b70}, 0xc056cb1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc027d94a80}, {0x4e9d7a0, 0xc035820200}, 0xc0b9208880, 0xc052a94558) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc027d94a80}, 0xc063f0c7e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0d995ed80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0d995ed80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809218 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035680, {0x63c85b8, 0xc021caeff0}, 0xc014c9a960, 0x1, 0xc02654b670, 0x519bbc, 0xc0488cb040, 0xc05ef9a540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01eabe900, {0x63c85b8, 0xc021caeff0}, 0xc01eabe938, 0xc02654b770, 0x41, 0x60, 0xffffffffffffffff, 0xc0488cb040, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035680, {0x63c85b8, 0xc021caeff0}, 0xc0488cae00, {{{{0xc038c6fd40, 0x3}, {0xc00c90fc20, 0x4}}, {{0xc07952aea0, 0x8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc021caeff0}, 0x63521a0, 0x57084ee0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd6d0, {0x63c85b8, 0xc021caeff0}, {0xc08eb95b00, {0x1743011009c19e32, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd6d0, {0x63c85b8, 0xc021caeff0}, 0xc02f7dfc20, 0xc02654bfb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc021caeff0}, 0x660089, {0xc08eb95b00, {0x1743011009c19e32, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc021caeff0}, 0xc04b09d800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011009c19e32, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011009c19e32, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021caef60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021caef60}, {0xc07e20f9c0, 0x660089}, 0xc07e20f9c8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021caef60}, {0xc00102e4e0}, 0xc04b09d700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021caeea0}, 0xc04b09d700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x63c85b8, 0xc07cd34960}, {0x7fe422a51230, 0xc05af62500}, 0xc0648aa5a0, {0xc0d3eb0900, 0x3, 0x4}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x24d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x63c85b8, 0xc07cd34960}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812819 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc05c285890}, 0xc03a781fe0, 0x1, 0xc03e4990a8, 0x519bbc, 0xc0117acde0, 0xc016da7920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05c285890}, 0x120, 0xc03e499230, 0x117acd80, 0x0, 0x660069, 0xc0117acde0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc05c285890}, 0xc0117acd80, {{{{0x0, 0x0}, {0xc028973680, 0x40}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05c285890}, 0x6352410, 0x3c90c588) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc05c285890}, {0xc0117acb40, {0x174301107f056b0c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc05c285890}, 0xc023799ea0, 0xc03e4999f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05c285890}, 0x660089, {0xc0117acb40, {0x174301107f056b0c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc05c285890}, 0xc02664f200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f056b0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f056b0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05c2857d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c2857d0}, {0xc03e49b3f8, 0x660089}, 0xc03e49b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05c2857d0}, {0x31342d346230622d}, 0xc02664f000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05c285740}, 0xc02664f000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}, 0xc03d9ad660, 0xc023415c08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}, 0x3, 0xc048fdeac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03e49b980}}}, {0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}, 0xc03d9ad660, 0xc048fdeac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05c285740}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c285740}, {0x40, 0xc03e49bb70}, 0xc03e49bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05c285740}, {0x4e9d7a0, 0xc02664f000}, 0xc03d9ad660, 0xc023415c08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05c285740}, 0xc092bf2600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0117ac900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0117ac900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806055 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030897c00, {0x63c85b8, 0xc01fd53020}, 0xc058ae8f00, 0xc0c91203c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030897c00, {0x63c85b8, 0xc01fd53020}, 0xc058ae8f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104b27194c, 0x0, 0x0}, 0xc075619330, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104b27194c, 0x0, 0x0}, 0xc075619330, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fd52f30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fd52f30}, {0xc005be7918, 0x660089}, 0xc005be7920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fd52f30}, {0xc00102e4e0}, 0xc058ae8e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fd52e70}, 0xc058ae8e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc02e4b1140}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc01fd52cc0}, {0xc00dfd36b0, 0x24}, 0x46c7c0ec6, 0xc088194d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x10, 0xc0110d1c78}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc01fd52cc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802887 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035680, {0x63c85b8, 0xc06b7d6f60}, 0xc05eae1e18, 0x1, 0xc0664d50a8, 0x519bbc, 0xc01e75e720, 0xc0488cae60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06b7d6f60}, 0x120, 0xc0664d5230, 0x1e75e6c0, 0x0, 0x660069, 0xc01e75e720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035680, {0x63c85b8, 0xc06b7d6f60}, 0xc01e75e6c0, {{{{0x0, 0x0}, {0xc03caefb00, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06b7d6f60}, 0x63526b0, 0x561afe38) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd6d0, {0x63c85b8, 0xc06b7d6f60}, {0xc06b7a1320, {0x174301102698c733, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd6d0, {0x63c85b8, 0xc06b7d6f60}, 0xc03d1ab040, 0xc0664d5558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b7d6f60}, 0xc03d1ab040, {0xc06b7a1320, {0x174301102698c733, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc06b7d6f60}, 0xc0ad363200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102698c733, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102698c733, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b7d6ed0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b7d6ed0}, {0xc0664d73f8, 0x660089}, 0xc0664d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b7d6ed0}, {0x4ed3465}, 0xc0ad363000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b7d6e40}, 0xc0ad363000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}, 0xc06bf507e0, 0xc02ee07b48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}, 0x3, 0xc062a54400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0664d7980}}}, {0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}, 0xc06bf507e0, 0xc062a54400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b7d6e40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b7d6e40}, {0x40, 0xc0664d7b70}, 0xc0664d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b7d6e40}, {0x4e9d7a0, 0xc0ad363000}, 0xc06bf507e0, 0xc02ee07b48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b7d6e40}, 0xc044fbf800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06b7a10e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06b7a10e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808169 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc06f432c90}, 0xc04a5f75c0, 0x1, 0x0, 0x1, 0xc050baca80, 0xc07854d020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06f432c90}, 0x120, 0xc00e80a728, 0x50baca20, 0x0, 0x660069, 0xc050baca80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc06f432c90}, 0xc050baca20, {{{{0x0, 0x0}, {0xc03ce90480, 0x1b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06f432c90}, 0x6352410, 0x2ad6f7d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc06f432c90}, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc06f432c90}, 0xc0255e1c20, 0xc082d14ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f432c90}, 0x660089, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc06f432c90}, 0xc084bb4c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f432c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f432c00}, {0xc0036ac8f0, 0x660089}, 0xc0036ac8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f432c00}, {0xc00102e4e0}, 0xc084bb4b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f432b40}, 0xc084bb4b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06f432810}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804808 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc082aa94a0}, 0xc0420ad788, 0x1, 0x4ed5f72, 0x7fdc6b09b98c, 0xc06b795140, 0xc055f37c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc082aa94a0}, 0x120, 0xc095eeb230, 0x6b7950e0, 0xc01262ab60, 0x18, 0xc06b795140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc082aa94a0}, 0xc06b7950e0, {{{{0x0, 0x0}, {0xc09298fd40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc082aa94a0}, 0x6352410, 0x199f0470) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc082aa94a0}, {0xc06b794fc0, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc082aa94a0}, 0xc074dced20, 0xc095eeb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc082aa94a0}, 0x660089, {0xc06b794fc0, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc082aa94a0}, 0xc0ae80ca00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc082aa9410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082aa9410}, {0xc095eed3f8, 0x660089}, 0xc095eed400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc082aa9410}, {0xc02844b590}, 0xc0ae80c800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc082aa9380}, 0xc0ae80c800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}, 0xc023048e40, 0xc0420ad758) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}, 0x3, 0xc00ecc6900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc095eed980}}}, {0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}, 0xc023048e40, 0xc00ecc6900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc082aa9380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082aa9380}, {0x40, 0xc095eedb70}, 0xc095eedab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}, 0x203003, 0x203003) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc082aa9380}, {0x4e9d7a0, 0xc0ae80c800}, 0xc023048e40, 0xc0420ad758) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc082aa9380}, 0xc06c737e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af93b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af93b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219789782 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc01fbf8e90, 0x2f) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc078274a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0782750e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08fa8b300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08fa8b300, {0x63c8510, 0xc01fbf8ec0}, 0xc0766b5dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0199af960) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc01fbf8ec0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0199af960) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802066 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc03b562510}, 0xc039706f00, 0x1, 0x0, 0x1, 0xc0389a4060, 0xc036f66060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03b562510}, 0x1e0, 0xc051098728, 0x389a4000, 0x0, 0x660069, 0xc0389a4060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc03b562510}, 0xc0389a4000, {{{{0x0, 0x0}, {0xc0587a9680, 0x34}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03b562510}, 0xc00db42fb0, 0x76588920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc03b562510}, {0xc0435b85a0, {0x174301101ebd1d6a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc03b562510}, 0xc01c752dc0, 0xc076588ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03b562510}, 0x660089, {0xc0435b85a0, {0x174301101ebd1d6a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03b562510}, 0xc0d383fa00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ebd1d6a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ebd1d6a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03b562420}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b562420}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03b562420}, {0xc00102e4e0}, 0xc0d383f900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03b5623c0}, 0xc0d383f900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc072be0c00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802719 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b9bd400, {0x63c85b8, 0xc0147bec30}, 0xc05ed1a500, 0xc0645825a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9bd400, {0x63c85b8, 0xc0147bec30}, 0xc05ed1a500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025600c09, 0x0, 0x0}, 0xc01e8f8320, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025600c09, 0x0, 0x0}, 0xc01e8f8320, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0147be8a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0147be8a0}, {0xc063e173f8, 0x660089}, 0xc063e17400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0147be8a0}, {0xc0015221e0}, 0xc05ed1a300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0147be630}, 0xc05ed1a300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}, 0xc0135d8500, 0xc042d4be78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}, 0x3, 0xc05511be40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc063e17980}}}, {0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}, 0xc0135d8500, 0xc05511be40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0147be630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0147be630}, {0x40, 0xc063e17b70}, 0xc063e17ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0147be630}, {0x4e9d7a0, 0xc05ed1a300}, 0xc0135d8500, 0xc042d4be78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0147be630}, 0xc027e42cc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0599b59e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0599b59e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801774 [runnable]: sync.runtime_SemacquireMutex(0xc097382690, 0x8f, 0x9bb12c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc097382538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc097382560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc068142000, {0xffffffffffffff15, 0xc0973826c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc068142000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc068142000, {0xc00bc4e1e0, 0x32, 0xa0}, {0xc0145ee400, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0be259298, {0xc00bc4e1e0, 0x32, 0xa0}, {0xc0145ee400, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0be257bc0, {0xc0145ee400, 0x77, 0xa0}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0be257bc0, {0xc00bc4e1e0, 0x77, 0xc0145ee400}, {0xc0145ee400, 0xc02e65ec80, 0x31}, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0be257900, {0xc0145ee400, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc07716b9a8, {{0xc02e65ec80, 0x31, 0x80}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc056d8fb00, {{0xc00f90f350, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05287e400, {0x63c85b8, 0xc0500666f0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0500666f0}, {0x64a6560, 0xc056d8fb00}, {0xc00f90f350, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0500666f0}, {0x6466ed8, 0xc07716b700}, {0xc00f90f350, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0173796f8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc050c03400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020e15c00, {0x63c85b8, 0xc0500666f0}, 0xc050c03400, 0xc06f64e6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020e15c00, {0x63c85b8, 0xc0500666f0}, 0xc050c03400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050066660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050066660}, {0xc0ad8388f0, 0x660089}, 0xc0ad8388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050066660}, {0xc00102e4e0}, 0xc050c03000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050066600}, 0xc050c03000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618caf90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811930 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc068306e70}, 0xc04479bd28, 0x1, 0xc0374570a8, 0x519bbc, 0xc060da6720, 0xc03d6f2cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc068306e70}, 0x120, 0xc037457230, 0x60da66c0, 0x0, 0x660069, 0xc060da6720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc068306e70}, 0xc060da66c0, {{{{0x0, 0x0}, {0xc063c79440, 0x30}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc068306e70}, 0x6352410, 0x55c36298) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc068306e70}, {0xc060da65a0, {0x1743011077a7adfe, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc068306e70}, 0xc00854f0e0, 0xc0374579f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc068306e70}, 0x660089, {0xc060da65a0, {0x1743011077a7adfe, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc068306e70}, 0xc08461bc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077a7adfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077a7adfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc068306de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc068306de0}, {0xc0374593f8, 0x660089}, 0xc037459400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc068306de0}, {0x7fdd4a38c988}, 0xc08461ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc068306d20}, 0xc08461ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}, 0xc07af6ff00, 0xc03e62f920) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}, 0x3, 0xc05c1eb280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc037459980}}}, {0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}, 0xc07af6ff00, 0xc05c1eb280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc068306d20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc068306d20}, {0x40, 0xc037459b70}, 0xc037459ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc068306d20}, {0x4e9d7a0, 0xc08461ba00}, 0xc07af6ff00, 0xc03e62f920) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc068306d20}, 0xc055315bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060da6480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060da6480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800607 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f6ea000, {0x63c85b8, 0xc01a971d10}, 0xc055d65a00, 0xc075ed4a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f6ea000, {0x63c85b8, 0xc01a971d10}, 0xc055d65a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110159679d1, 0x0, 0x0}, 0xc0b69c0780, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110159679d1, 0x0, 0x0}, 0xc0b69c0780, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01a971c80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01a971c80}, {0xc076589918, 0x660089}, 0xc076589920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01a971c80}, {0xc00102e4e0}, 0xc055d65900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01a971c20}, 0xc055d65900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc02e5f8b40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc01a971b90}, {0xc00dfd36b0, 0x24}, 0x478beaa47, 0xc015271d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc01a971b90}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809196 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc07bc0fef0}, 0xc06292eee8, 0x1, 0xc01ac270a8, 0x519bbc, 0xc06e4b0cc0, 0xc0610ce180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07bc0fef0}, 0x120, 0xc01ac27230, 0x6e4b0c60, 0x0, 0x660069, 0xc06e4b0cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc07bc0fef0}, 0xc06e4b0c60, {{{{0x0, 0x0}, {0xc08fa2f440, 0xa5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07bc0fef0}, 0x6352410, 0x1b308330) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc07bc0fef0}, {0xc06e4b0b40, {0x1743011063a956e0, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc07bc0fef0}, 0xc0515c3ae0, 0xc01ac279f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07bc0fef0}, 0x660089, {0xc06e4b0b40, {0x1743011063a956e0, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc07bc0fef0}, 0xc05bac0900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063a956e0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063a956e0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07bc0fe60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bc0fe60}, {0xc01ac293f8, 0x660089}, 0xc01ac29400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07bc0fe60}, {0x7fe3bc0b0048}, 0xc05bac0700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07bc0fdd0}, 0xc05bac0700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}, 0xc03f4995e0, 0xc042c25638) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}, 0x3, 0xc04f1cb780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01ac29980}}}, {0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}, 0xc03f4995e0, 0xc04f1cb780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07bc0fdd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bc0fdd0}, {0x40, 0xc01ac29b70}, 0xc01ac29ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07bc0fdd0}, {0x4e9d7a0, 0xc05bac0700}, 0xc03f4995e0, 0xc042c25638) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07bc0fdd0}, 0xc03de0b920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052b78fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052b78fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808781 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03bd19180, {0x63c85b8, 0xc041074d50}, 0xc02f7dd5a8, 0x1, 0xc0174690a8, 0x519bbc, 0xc08c7848a0, 0xc04a8c1500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05907c000, {0x63c85b8, 0xc041074d50}, 0xc05907c030, 0xc0174691a8, 0x45, 0x60, 0xffffffffffffffff, 0xc08c7848a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03bd19180, {0x63c85b8, 0xc041074d50}, 0xc08c7847e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc073d50000, 0x9}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041074d50}, 0x63521a0, 0x1fa2e648) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc067985680, {0x63c85b8, 0xc041074d50}, {0xc08c7846c0, {0x1743010fddb2183d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc067985680, {0x63c85b8, 0xc041074d50}, 0xc0307a7180, 0xc0174699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041074d50}, 0x660089, {0xc08c7846c0, {0x1743010fddb2183d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc041074d50}, 0xc0450c9000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fddb2183d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fddb2183d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041074cc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041074cc0}, {0xc01746b3f8, 0x660089}, 0xc01746b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041074cc0}, {0x7fe3944b2170}, 0xc0450c8d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041074c30}, 0xc0450c8d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}, 0xc05b3b8620, 0xc003ce3218) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}, 0x3, 0xc03df82100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01746b980}}}, {0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}, 0xc05b3b8620, 0xc03df82100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041074c30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041074c30}, {0x40, 0xc01746bb70}, 0xc01746bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041074c30}, {0x4e9d7a0, 0xc0450c8d00}, 0xc05b3b8620, 0xc003ce3218) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041074c30}, 0xc05a9736e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08c7845a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08c7845a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812259 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc03f780a20}, 0xc016f61260, 0x1, 0xc033a0f0a8, 0x519bbc, 0xc035281a40, 0xc0576c7b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03f780a20}, 0x120, 0xc033a0f230, 0x352819e0, 0x0, 0x660069, 0xc035281a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc03f780a20}, 0xc0352819e0, {{{{0x0, 0x0}, {0xc02875a900, 0x35}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03f780a20}, 0x6352410, 0x65f8a4f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc03f780a20}, {0xc0352818c0, {0x174301107a113c4d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc03f780a20}, 0xc07b1e9c20, 0xc033a0f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03f780a20}, 0x660089, {0xc0352818c0, {0x174301107a113c4d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc03f780a20}, 0xc04983c600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a113c4d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a113c4d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03f780990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f780990}, {0xc033a113f8, 0x660089}, 0xc033a11400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03f780990}, {0xc0015221e0}, 0xc04983c400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03f780900}, 0xc04983c400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}, 0xc03294f460, 0xc04c38e090) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}, 0x3, 0xc037015600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc033a11980}}}, {0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}, 0xc03294f460, 0xc037015600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03f780900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f780900}, {0x40, 0xc033a11b70}, 0xc033a11ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03f780900}, {0x4e9d7a0, 0xc04983c400}, 0xc03294f460, 0xc04c38e090) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03f780900}, 0xc035310ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0352817a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0352817a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812427 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc03100b400, {0x63c85b8, 0xc04f5695c0}, 0xc070375c00, 0xc075440780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc03100b400, {0x63c85b8, 0xc04f5695c0}, 0xc070375c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109e396f44, 0x0, 0x0}, 0xc033eafa10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109e396f44, 0x0, 0x0}, 0xc033eafa10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f569530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f569530}, {0xc07d93f918, 0x660089}, 0xc07d93f920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f569530}, {0xc00102e4e0}, 0xc070375b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f5694d0}, 0xc070375b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06123b380}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04f569440}, {0xc00dfd36b0, 0x24}, 0x4610dd8c1, 0xc02790dd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc0b1f181e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04f569440}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801180 [runnable]: sync.runtime_SemacquireMutex(0xc0791da690, 0x8f, 0x7fdaf1228000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0791da538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0791da560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc08dd4fb00, {0xffffffffffffff15, 0xc0791da6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc08dd4fb00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc08dd4fb00, {0xc07954a580, 0x32, 0x74}, {0xc02f863d80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc079ff3998, {0xc07954a580, 0x32, 0x74}, {0xc02f863d80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc079ff22c0, {0xc02f863d80, 0x74, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc079ff22c0, {0xc07954a580, 0x74, 0xc02f863d80}, {0xc02f863d80, 0xc07a64a2c0, 0x31}, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc079ff2000, {0xc02f863d80, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0578b98a8, {{0xc07a64a2c0, 0x31, 0xb0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0618da500, {{0xc0b8c29b90, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0484b0800, {0x63c85b8, 0xc076faff50}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc076faff50}, {0x64a6560, 0xc0618da500}, {0xc0b8c29b90, 0x20907, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc076faff50}, {0x6466ed8, 0xc0578b9600}, {0xc0b8c29b90, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc042f81368}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05b5d6700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c25400, {0x63c85b8, 0xc076faff50}, 0xc05b5d6700, 0xc04cf17b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c25400, {0x63c85b8, 0xc076faff50}, 0xc05b5d6700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076fafec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076fafec0}, {0xc0936988f0, 0x660089}, 0xc0936988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076fafec0}, {0xc00102e4e0}, 0xc05b5d6600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076fafe60}, 0xc05b5d6600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc078273800}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805660 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035040, {0x63c85b8, 0xc067bb3f20}, 0xc04683fe18, 0x1, 0x0, 0x1, 0xc06bd42cc0, 0xc076a74180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc067bb3f20}, 0x120, 0xc00598a728, 0x6bd42c60, 0x0, 0x660069, 0xc06bd42cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035040, {0x63c85b8, 0xc067bb3f20}, 0xc06bd42c60, {{{{0x0, 0x0}, {0xc01f5d0fc0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc067bb3f20}, 0x6352410, 0x52e790a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd3b0, {0x63c85b8, 0xc067bb3f20}, {0xc06bd42b40, {0x174301104a2798a9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd3b0, {0x63c85b8, 0xc067bb3f20}, 0xc02e903a40, 0xc057c74ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067bb3f20}, 0x660089, {0xc06bd42b40, {0x174301104a2798a9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc067bb3f20}, 0xc04cfc8600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104a2798a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104a2798a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067bb3e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067bb3e90}, {0xc09363c8f0, 0x660089}, 0xc09363c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067bb3e90}, {0xc00102e4e0}, 0xc04cfc8500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067bb3e30}, 0xc04cfc8500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc067bb3c50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808498 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc067e0f950}, 0xc02a5757d0, 0x1, 0x0, 0x1, 0xc05ab87da0, 0xc0556983c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc067e0f950}, 0x120, 0xc011d76728, 0x5ab87d40, 0x0, 0x660069, 0xc05ab87da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc067e0f950}, 0xc05ab87d40, {{{{0x0, 0x0}, {0xc029939680, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc067e0f950}, 0x6352410, 0x53d3d360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc067e0f950}, {0xc05ab87b00, {0x174301105bff062f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc067e0f950}, 0xc0713046e0, 0xc07ce42ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067e0f950}, 0x660089, {0xc05ab87b00, {0x174301105bff062f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc067e0f950}, 0xc015105a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067e0f8c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067e0f8c0}, {0xc002fe68f0, 0x660089}, 0xc002fe68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067e0f8c0}, {0xc00102e4e0}, 0xc015105900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067e0f860}, 0xc015105900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc067e0f680}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804933 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc077cd1e60}, 0xc02f43e0a8, 0x1, 0x0, 0x1, 0xc06eb5acc0, 0xc0596003c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077cd1e60}, 0x120, 0xc0798a8728, 0x6eb5ac60, 0xc00bd06ea0, 0x18, 0xc06eb5acc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc077cd1e60}, 0xc06eb5ac60, {{{{0x0, 0x0}, {0xc01fe83440, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077cd1e60}, 0x6352410, 0x71c3e3d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc077cd1e60}, {0xc06eb5ab40, {0x174301103269927a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc077cd1e60}, 0xc02335c500, 0xc064c36ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077cd1e60}, 0x660089, {0xc06eb5ab40, {0x174301103269927a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc077cd1e60}, 0xc08ca37100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103269927a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103269927a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077cd1dd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077cd1dd0}, {0xc002fe68f0, 0x660089}, 0xc002fe68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077cd1dd0}, {0xc00102e4e0}, 0xc08ca36f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077cd1d70}, 0xc08ca36f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc077cd1b60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811654 [runnable]: sync.runtime_SemacquireMutex(0xc033eed198, 0x8f, 0x7fe3662ac9c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc033eed040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x21, 0x18, 0x21, 0xc033eed068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04fc8f200, {0xffffffffffffff15, 0xc033eed1c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc04fc8f200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04fc8f200, {0xc00f6f54d0, 0x1f, 0x90}, {0xc02deadc80, 0x1f, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc088492d98, {0xc00f6f54d0, 0x1f, 0x90}, {0xc02deadc80, 0x1f, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0884916c0, {0xc02deadc80, 0x74, 0x90}, 0x1f, 0x1f) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0884916c0, {0xc00f6f54d0, 0x74, 0xc02deadc80}, {0xc02deadc80, 0xc0409759a0, 0x1e}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc088491400, {0xc02deadc80, 0x1f, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06c035328, {{0xc0409759a0, 0x1e, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0375ee600, {{0xc05ed28108, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc082a78800, {0x63c85b8, 0xc06550f3b0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06550f3b0}, {0x64a6560, 0xc0375ee600}, {0xc05ed28108, 0x0, 0x18}, {0x1743011077a34578, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06550f3b0}, {0x6466ed8, 0xc06c035080}, {0xc05ed28108, 0x17, 0x18}, {0x1743011077a34578, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0672e97d0}, {{0x1743011077a34578, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011077a34578, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc066efec00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174301109570aa78, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02fe65000, {0x63c85b8, 0xc06550f3b0}, 0xc066efec00, 0xc0622be3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fe65000, {0x63c85b8, 0xc06550f3b0}, 0xc066efec00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077a34578, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077a34578, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06550f320}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06550f320}, {0xc033ef33f8, 0x660089}, 0xc033ef3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06550f320}, {0x0}, 0xc066efea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06550f200}, 0xc066efea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}, 0xc058856c20, 0xc04aa6f7b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}, 0x3, 0xc06c9abac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc033ef3980}}}, {0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}, 0xc058856c20, 0xc06c9abac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06550f200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06550f200}, {0x40, 0xc033ef3b70}, 0xc033ef3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06550f200}, {0x4e9d7a0, 0xc066efea00}, 0xc058856c20, 0xc04aa6f7b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06550f200}, 0xc05a98c2a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc011e805a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc011e805a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803883 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc06e8f5610, 0x3c) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc04467b500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc04467bb60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0d560f300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0d560f300, {0x63c8510, 0xc06e8f5680}, 0xc073097dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0534b1f40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06e8f5680}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0534b1f40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219794468 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0476ca780}, 0xc033d18cb0, {0x646cc48, 0xc0476ca7e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0476ca780}, 0xedb7a834e, 0xc070bcc6c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0476ca780}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0393db170}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc033d18ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0393db170}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219185883 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc047776d80) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0517ad080, {0x4d83cc0, 0xc079664180}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x11b) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc047776b40, 0xc08a3a8f68, 0xc08a3a8f38) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc047776b40, {0x4d83cc0, 0xc079664180}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc026c7b110}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc041655000}, {0x4d83cc0, 0xc079664180}, 0x2, {0xc012856840, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc026c7b110}, {0x4f6406d, 0x7fe457d81688}, {0x4e9d7a0, 0xc041655000}, {0x4d83cc0, 0xc079664180}, 0xac5d5ba1edb7d4b, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x9ef2599772ae041, {0x63c85b8, 0xc026c7b110}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc041655000}, {0x4d83cc0, 0xc079664180}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc03ad19330, {0x63c85b8, 0xc026c7b110}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc03ad19330}}, {0x63c85b8, 0xc026c7b110}, 0xc041655000, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1c, 0x1c, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc03cf66c30, {0x63c85b8, 0xc026c7b0b0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc03cf66c30, {0x63c85b8, 0xc026c7b0b0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc08ba7b130, {0x63c85b8, 0xc026c7b0b0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc077f36700, {0x63c85b8, 0xc026c7b0b0}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0846cb4a0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc06624da40) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc02a581800) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).next(0xc05b6c1ce0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:374 +0x7d github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc05b6c1ce0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:189 +0x32c github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc06624db00) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc026c7af90) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/is_null_ops.eg.go:226 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc015143800) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc0757653f0, {0x63c8510, 0xc06624da00}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0516ccdc0, {0x63c8510, 0xc06624da00}, 0xc09ad01810) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc06624d600}, 0xc05fe96700, 0xc09ad01810, 0xc075c09f00, 0xc0d05e2a80, 0xc0c902fce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc06624d600}, 0xc0c902f9f0, 0xc05fe96700, 0xc0a546b140, {{0x63ca340, 0xc075c09e80}, 0x0}, 0xc0d05e2a80) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0c902f300, {0x63c8510, 0xc06624d600}, 0xc0c902f9f0, 0xc06624d600, {0x7fe42b790cf0, 0xc0a546b140}, 0x0, 0xc02a580d78) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0c902f300, {0x63c8510, 0xc06624d600}, 0xc0c902f9f0, {0x7fe42b790cf0, 0xc0a546b140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0c902f300, {0x63c85b8, 0xc01da161b0}, {{0x63f8d78, 0xc056eb4500}, {0xc05587d00d, 0xb4}, 0x4, 0x1}, 0xc042adb320, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0c902f300, {0x63c85b8, 0xc01da161b0}, {{0x63f8d78, 0xc056eb4500}, {0xc05587d00d, 0xb4}, 0x4, 0x1}, 0xc042adb320, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0c902f300, {0x63c85b8, 0xc01da161b0}, {0xc042adb320, {0xc06624d5c0, 0x4, 0x4}, {0xc0066b78d8, 0xb, 0xb}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc07849bbbb, 0x0}, 0x0, {0x350bd8ff, 0xedb7a834a, 0x0}, 0x1}, 0xc0c902f300, 0xc08a3af980, 0xc08a3af970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0c902f300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0c902f300, {0x63c8510, 0xc0cdc29000}, 0xc04d09fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc054206b50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0cdc29000}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc054206b50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811675 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc020840e70}, 0xc04170ec48, 0x1, 0xc033f330a8, 0x519bbc, 0xc01cb543c0, 0xc00d5a8720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc020840e70}, 0x120, 0xc033f33230, 0x1cb54360, 0x0, 0x660069, 0xc01cb543c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc020840e70}, 0xc01cb54360, {{{{0x0, 0x0}, {0xc0b4bac6c0, 0x2a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc020840e70}, 0x6352410, 0x3f55a8f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc020840e70}, {0xc01cb54120, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc020840e70}, 0xc0509359a0, 0xc033f339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc020840e70}, 0x660089, {0xc01cb54120, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc020840e70}, 0xc0689b2100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020840de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020840de0}, {0xc033f353f8, 0x660089}, 0xc033f35400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020840de0}, {0x7fe3f1fa9f40}, 0xc06e151e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020840d50}, 0xc06e151e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}, 0xc086728c80, 0xc04170fa28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}, 0x3, 0xc0773fd9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc033f35980}}}, {0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}, 0xc086728c80, 0xc0773fd9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020840d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020840d50}, {0x40, 0xc033f35b70}, 0xc033f35ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020840d50}, {0x4e9d7a0, 0xc06e151e00}, 0xc086728c80, 0xc04170fa28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020840d50}, 0xc0c45dce40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc010a39e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc010a39e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807499 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc04c47fdd0}, 0xc047937200, 0x1, 0xc0061c10a8, 0x519bbc, 0xc0557ed140, 0xc060e2f8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04c47fdd0}, 0x1e0, 0xc0061c1230, 0x557ed0e0, 0x0, 0x660069, 0xc0557ed140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc04c47fdd0}, 0xc0557ed0e0, {{{{0x0, 0x0}, {0xc0561ed680, 0x32}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04c47fdd0}, 0x63521a0, 0x4f455788) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc04c47fdd0}, {0xc08fdce5a0, {0x17430110576c2c1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc04c47fdd0}, 0xc06810ea00, 0xc0061c19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c47fdd0}, 0x660089, {0xc08fdce5a0, {0x17430110576c2c1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc04c47fdd0}, 0xc0991f5400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110576c2c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110576c2c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c47fd40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c47fd40}, {0xc0061c33f8, 0x660089}, 0xc0061c3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c47fd40}, {0x0}, 0xc0991f5200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c47fcb0}, 0xc0991f5200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}, 0xc0315b5aa0, 0xc031bf8ed0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}, 0x3, 0xc063172dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0061c3980}}}, {0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}, 0xc0315b5aa0, 0xc063172dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04c47fcb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c47fcb0}, {0x40, 0xc0061c3b70}, 0xc0061c3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04c47fcb0}, {0x4e9d7a0, 0xc0991f5200}, 0xc0315b5aa0, 0xc031bf8ed0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04c47fcb0}, 0xc04f3889c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08e5a6900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08e5a6900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806151 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc0648422a0}, 0xc06fcccf00, 0xc0ce913a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0648422a0}, 0xc06fcccf00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104ba6f35b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104ba6f35b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064842210}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064842210}, {0xc0052a93f8, 0x660089}, 0xc0052a9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064842210}, {0x3d7756454c4c6c33}, 0xc06fcccd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064842180}, 0xc06fcccd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}, 0xc078eae160, 0xc08d7fa168) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}, 0x3, 0xc06debab40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0052a9980}}}, {0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}, 0xc078eae160, 0xc06debab40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc064842180}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064842180}, {0x40, 0xc0052a9b70}, 0xc0052a9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc064842180}, {0x4e9d7a0, 0xc06fcccd00}, 0xc078eae160, 0xc08d7fa168) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc064842180}, 0xc0531931a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03e484c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03e484c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806438 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d9e00, {0x63c85b8, 0xc08a2a2300}, 0xc090cf9578, 0x1, 0xc00c5f9ff0, 0x519bbc, 0xc062d91380, 0xc06e943380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08a2a2300}, 0x120, 0xc00c5fa178, 0x62d91320, 0x0, 0x660069, 0xc062d91380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d9e00, {0x63c85b8, 0xc08a2a2300}, 0xc062d91320, {{{{0x0, 0x0}, {0xc0107bb8c0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08a2a2300}, 0x6352410, 0x321a0748) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0215a8b40, {0x63c85b8, 0xc08a2a2300}, {0xc02cd7f7a0, {0x174301104eac03bc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0215a8b40, {0x63c85b8, 0xc08a2a2300}, 0xc050d0e640, 0xc07685a938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08a2a2300}, 0x660089, {0xc02cd7f7a0, {0x174301104eac03bc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0215f1400, {0x63c85b8, 0xc08a2a2300}, 0xc09d04a800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104eac03bc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104eac03bc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08a2a2270}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a2a2270}, {0xc07685c340, 0x660089}, 0xc07685c348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08a2a2270}, {0xc00102e4e0}, 0xc09d04a700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08a2a2210}, 0xc09d04a700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc083279b60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812469 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc072bf13e0}, 0xc041c39cb0, 0x1, 0xc03c0ad6d8, 0x519bbc, 0xc082445e00, 0xc06c1f0d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01699be80, {0x63c85b8, 0xc072bf13e0}, 0xc040092480, 0xc03c0ad7d8, 0x82445e00, 0x0, 0x0, 0xc082445e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc072bf13e0}, 0xc082445d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0570a6900, 0x4c}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc072bf13e0}, 0x6352260, 0x33a92d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc072bf13e0}, {0xc082445c20, {0x1743011061978db5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc072bf13e0}, 0xc02ad1f680, 0xc03c0ae020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc072bf13e0}, 0x660089, {0xc082445c20, {0x1743011061978db5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc072bf13e0}, 0xc01a4e5d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061978db5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061978db5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072bf1320}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072bf1320}, {0xc07cdf5a28, 0x660089}, 0xc07cdf5a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072bf1320}, {0xc00102e4e0}, 0xc01a4e5600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072bf12c0}, 0xc01a4e5600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0459fc2d0, {0x63c85b8, 0xc072bf1230}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0459fc2d0, {0x63c85b8, 0xc072bf1230}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc072bf1230}, {0x4f67ad2, 0x6446ff8}, 0xc019549fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219799476 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0623ccdc0, {0x63c85b8, 0xc0496339e0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0b4607900, {0x63c85b8, 0xc0496339e0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0b4607900, {0x63c8510, 0xc041a0afc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc0b4607900, {0x63c8510, 0xc041a0afc0}, 0xc04ce10650, {0x7fe42b790cf0, 0xc091c6f140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0b4607900, {0x63c85b8, 0xc06b42dc50}, {{0x63f88f8, 0xc04ce10650}, {0xc0a63341df, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0b4607900, {0x63c85b8, 0xc06b42dc50}, {{0x63f88f8, 0xc04ce10650}, {0xc0a63341df, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc04ce10650}, {0xc0a63341df, 0x23}, 0x0, 0x0}, {0x28a2ceac, 0xedb7a8350, 0x0}, {0x28a2d3f0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0b4607900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0b4607900, {0x63c8510, 0xc0b36c3880}, 0xc07b0d7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc047053200) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0b36c3880}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc047053200) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219805447 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc025026dc0, {0x63c85b8, 0xc07a7ad7d0}, 0xc0314e57a0, 0x1, 0xc075f270a8, 0x519bbc, 0xc05f01d020, 0xc06e73aba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07a7ad7d0}, 0x120, 0xc075f27230, 0x5f01cfc0, 0xc010141040, 0x18, 0xc05f01d020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc025026dc0, {0x63c85b8, 0xc07a7ad7d0}, 0xc05f01cfc0, {{{{0x0, 0x0}, {0xc073da2b40, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07a7ad7d0}, 0x6352410, 0x71c3f650) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0234e8ff0, {0x63c85b8, 0xc07a7ad7d0}, {0xc05f01cea0, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0234e8ff0, {0x63c85b8, 0xc07a7ad7d0}, 0xc02536abe0, 0xc075f279f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a7ad7d0}, 0x660089, {0xc05f01cea0, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02505ec00, {0x63c85b8, 0xc07a7ad7d0}, 0xc01fb3b000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a7ad740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a7ad740}, {0xc075f293f8, 0x660089}, 0xc075f29400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a7ad740}, {0x7fe3262dccb8}, 0xc01fb3ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a7ad6b0}, 0xc01fb3ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}, 0xc03a016600, 0xc031fac150) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}, 0x3, 0xc065b2f080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075f29980}}}, {0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}, 0xc03a016600, 0xc065b2f080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a7ad6b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a7ad6b0}, {0x40, 0xc075f29b70}, 0xc075f29ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}, 0x203019, 0x203019) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a7ad6b0}, {0x4e9d7a0, 0xc01fb3ad00}, 0xc03a016600, 0xc031fac150) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a7ad6b0}, 0xc06cde14a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f70b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f70b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812585 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812716 [runnable]: sync.runtime_SemacquireMutex(0xc03fa96d80, 0x4e, 0xc00074b700) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0x118487c, 0xc00074b6c0, 0x7fd5806aa540) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x3d7924b, 0x76430b7, 0x7fd5806aa540, 0x7fe08bfa52e0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x4d866e, 0x9185600}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).init(0xc04f68db00, 0xc00e0b2580, {0xc036907500, 0x32, 0xc0}, {0xc0369075c0, 0x32, 0xc0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1229 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00e0b2580, {0xc036907500, 0x32, 0xc0}, {0xc0369075c0, 0x32, 0xc0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2157 +0xdb github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a528c0, 0x32, 0xc085344de8, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0xc06290a1c0, 0xc085344f00, 0x0, 0xc085344e00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc085344d98, 0xc0734106e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc085344d98, {0xc02f5b0b00, 0x32, 0x72}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:400 +0x97 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0853436c0, {0xc02f5b0b00, 0xc0, 0x72}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc0853436c0, {0xc02f5b0b00, 0x32, 0xc0369075c0}, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc085343400, {0xc02f5b0b00, 0x32, 0x72}, {0xc0077383c0, 0x34, 0xa0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04369bf78, {{0xc0acb45180, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0xc0077383c0, 0x33, 0xa0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:334 +0x173 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc091103700, {{0xc076005770, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc049625800, {{0xc076005770, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc049625800, {0x63c85b8, 0xc0505205d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc0760057a0, 0x2a, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.MVCCScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc0760057a0, 0x2a, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2504 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc049dac270}, {{0x174301108e847020, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:62 +0x41b github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301108e847020, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03486fc00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110ac51d520, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc03100b400, {0x63c85b8, 0xc0505205d0}, 0xc03486fc00, 0xc07bbecc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc03100b400, {0x63c85b8, 0xc0505205d0}, 0xc03486fc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108e847020, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108e847020, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050520540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050520540}, {0xc03fa9d3f8, 0x660089}, 0xc03fa9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050520540}, {0xa1fab9aae2a3080a}, 0xc03486f900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0505204b0}, 0xc03486f900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}, 0xc00abd2f80, 0xc017daeba0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}, 0x3, 0xc06b3d6680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03fa9d980}}}, {0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}, 0xc00abd2f80, 0xc06b3d6680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0505204b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0505204b0}, {0x40, 0xc03fa9db70}, 0xc03fa9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0505204b0}, {0x4e9d7a0, 0xc03486f900}, 0xc00abd2f80, 0xc017daeba0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0505204b0}, 0xc0289e7aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a46c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a46c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812005 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f7a4dc0, {0x63c85b8, 0xc06199cff0}, 0xc013d6ccd8, 0x1, 0xc034e870a8, 0x519bbc, 0xc065f5d5c0, 0xc03efd5da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06199cff0}, 0x120, 0xc034e87230, 0x65f5d560, 0x0, 0x660069, 0xc065f5d5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f7a4dc0, {0x63c85b8, 0xc06199cff0}, 0xc065f5d560, {{{{0x0, 0x0}, {0xc04e62f0e0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06199cff0}, 0x6352410, 0x33a923b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f663040, {0x63c85b8, 0xc06199cff0}, {0xc065f5d440, {0x174301107c36161b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f663040, {0x63c85b8, 0xc06199cff0}, 0xc03b3641e0, 0xc034e879f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06199cff0}, 0x660089, {0xc065f5d440, {0x174301107c36161b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f7c3000, {0x63c85b8, 0xc06199cff0}, 0xc06f31c100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107c36161b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107c36161b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06199cf30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06199cf30}, {0xc034e893f8, 0x660089}, 0xc034e89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06199cf30}, {0x7fe3262f55c0}, 0xc060cc9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06199cea0}, 0xc060cc9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}, 0xc0262c6820, 0xc013d6cfa8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}, 0x3, 0xc04e09ea00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034e89980}}}, {0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}, 0xc0262c6820, 0xc04e09ea00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06199cea0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06199cea0}, {0x40, 0xc034e89b70}, 0xc034e89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06199cea0}, {0x4e9d7a0, 0xc060cc9f00}, 0xc0262c6820, 0xc013d6cfa8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06199cea0}, 0xc063da7aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc065f5d320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc065f5d320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811242 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc021d78750}, 0xc024f62c00, 0xc050935ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc021d78750}, 0xc024f62c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011074268847, 0x0, 0x0}, 0xc05f3f93c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011074268847, 0x0, 0x0}, 0xc05f3f93c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021d78690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021d78690}, {0xc0182873f8, 0x660089}, 0xc018287400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021d78690}, {0x7265765f77736d72}, 0xc024f62a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021d783c0}, 0xc024f62a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}, 0xc040f5c3e0, 0xc049bf0180) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}, 0x3, 0xc0286a3980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc018287980}}}, {0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}, 0xc040f5c3e0, 0xc0286a3980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc021d783c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021d783c0}, {0x40, 0xc018287b70}, 0xc018287ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc021d783c0}, {0x4e9d7a0, 0xc024f62a00}, 0xc040f5c3e0, 0xc049bf0180) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc021d783c0}, 0xc026f6aba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06eb95560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06eb95560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809206 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01f1b2000, {0x63c85b8, 0xc043ba0c00}, 0xc0738dc6c0, 0x1, 0xc0132290a8, 0x519bbc, 0xc0788c0300, 0xc05299b3e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc057369560, {0x63c85b8, 0xc043ba0c00}, 0xc0788c0300, 0xc0132291a8, 0x44, 0x60, 0x0, 0xc0788c0300, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01f1b2000, {0x63c85b8, 0xc043ba0c00}, 0xc0788c0000, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0906b9320, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc043ba0c00}, 0x63521a0, 0x46e76f38) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e9c9d10, {0x63c85b8, 0xc043ba0c00}, {0xc087b57200, {0x1743010ff6423640, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e9c9d10, {0x63c85b8, 0xc043ba0c00}, 0xc065f2f540, 0xc0132299f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc043ba0c00}, 0x660089, {0xc087b57200, {0x1743010ff6423640, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f563000, {0x63c85b8, 0xc043ba0c00}, 0xc0af0c1d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff6423640, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff6423640, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043ba0b70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043ba0b70}, {0xc01322b3f8, 0x660089}, 0xc01322b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043ba0b70}, {0x0}, 0xc0af0c1a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043ba0a50}, 0xc0af0c1a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}, 0xc01d037f00, 0xc03748d6f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}, 0x3, 0xc0379262c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01322b980}}}, {0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}, 0xc01d037f00, 0xc0379262c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc043ba0a50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043ba0a50}, {0x40, 0xc01322bb70}, 0xc01322bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc043ba0a50}, {0x4e9d7a0, 0xc0af0c1a00}, 0xc01d037f00, 0xc03748d6f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc043ba0a50}, 0xc06f58d080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc087b570e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc087b570e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802475 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc025d5f400, {0x63c85b8, 0xc0247b3980}, 0xc059201d00, 0xc0732d6640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025d5f400, {0x63c85b8, 0xc0247b3980}, 0xc059201d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110254dcb55, 0x0, 0x0}, 0xc03ba34820, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110254dcb55, 0x0, 0x0}, 0xc03ba34820, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0247b38f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0247b38f0}, {0xc064b953f8, 0x660089}, 0xc064b95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0247b38f0}, {0x0}, 0xc059201b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0247b3830}, 0xc059201b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}, 0xc068275960, 0xc01ff39dd0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}, 0x3, 0xc0d43ef980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc064b95980}}}, {0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}, 0xc068275960, 0xc0d43ef980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0247b3830}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0247b3830}, {0x40, 0xc064b95b70}, 0xc064b95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0247b3830}, {0x4e9d7a0, 0xc059201b00}, 0xc068275960, 0xc01ff39dd0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0247b3830}, 0xc0543f4a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04732b8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04732b8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802801 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04aadf3e0}, 0xc03a855d10, 0x1, 0xc05ff430a8, 0x519bbc, 0xc03f60c600, 0xc03bf58f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04aadf3e0}, 0x1e0, 0xc05ff43230, 0x3f60c5a0, 0x0, 0x660069, 0xc03f60c600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04aadf3e0}, 0xc03f60c5a0, {{{{0x0, 0x0}, {0xc09ed7e6c0, 0xcf}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04aadf3e0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04aadf3e0}, {0xc04c4c85a0, {0x1743011022bb31b5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04aadf3e0}, 0xc070dd8fa0, 0xc05ff439f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04aadf3e0}, 0x660089, {0xc04c4c85a0, {0x1743011022bb31b5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04aadf3e0}, 0xc0781f2600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011022bb31b5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011022bb31b5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04aadf350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aadf350}, {0xc05ff453f8, 0x660089}, 0xc05ff45400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04aadf350}, {0x7fe13e89ce98}, 0xc0781f2400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04aadee40}, 0xc0781f2400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}, 0xc06264e8c0, 0xc057276d20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}, 0x3, 0xc06bdb76c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05ff45980}}}, {0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}, 0xc06264e8c0, 0xc06bdb76c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04aadee40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aadee40}, {0x40, 0xc05ff45b70}, 0xc05ff45ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04aadee40}, {0x4e9d7a0, 0xc0781f2400}, 0xc06264e8c0, 0xc057276d20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04aadee40}, 0xc057f14ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04c4c8480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04c4c8480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812603 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809603 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc067f73ef0}, 0xc06ac98ff0, 0x1, 0xc0049890a8, 0x519bbc, 0xc04842f020, 0xc04b293020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc067f73ef0}, 0x120, 0xc004989230, 0x4842efc0, 0x0, 0x660069, 0xc04842f020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc067f73ef0}, 0xc04842efc0, {{{{0x0, 0x0}, {0xc04b734c60, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc067f73ef0}, 0x6352410, 0x1fa2edf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc067f73ef0}, {0xc04842eea0, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc067f73ef0}, 0xc0738b3680, 0xc0049899f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067f73ef0}, 0x660089, {0xc04842eea0, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc067f73ef0}, 0xc023a53200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067f73e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067f73e60}, {0xc00498b3f8, 0x660089}, 0xc00498b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067f73e60}, {0x7fe2ff55a0e8}, 0xc023a53000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067f73dd0}, 0xc023a53000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}, 0xc085fd2780, 0xc0403e64f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}, 0x3, 0xc021bb6a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00498b980}}}, {0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}, 0xc085fd2780, 0xc021bb6a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc067f73dd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067f73dd0}, {0x40, 0xc00498bb70}, 0xc00498bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc067f73dd0}, {0x4e9d7a0, 0xc023a53000}, 0xc085fd2780, 0xc0403e64f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc067f73dd0}, 0xc0314ebd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc088145b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc088145b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804533 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc027ab18f0}, 0xc074a63d28, 0x1, 0xc074f770a8, 0x519bbc, 0xc058d9b140, 0xc05859f260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc027ab18f0}, 0x120, 0xc074f77230, 0x58d9b0e0, 0x0, 0x660069, 0xc058d9b140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc027ab18f0}, 0xc058d9b0e0, {{{{0x0, 0x0}, {0xc01fe83440, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc027ab18f0}, 0x6352410, 0x17e2e680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc027ab18f0}, {0xc058d9afc0, {0x174301103274ea0c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc027ab18f0}, 0xc0742b1cc0, 0xc074f779f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc027ab18f0}, 0x660089, {0xc058d9afc0, {0x174301103274ea0c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc027ab18f0}, 0xc0618da100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103274ea0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103274ea0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027ab1860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027ab1860}, {0xc074f793f8, 0x660089}, 0xc074f79400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027ab1860}, {0x7fe2ff4f63f0}, 0xc0d7eebf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027ab17d0}, 0xc0d7eebf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}, 0xc073550280, 0xc042fba720) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}, 0x3, 0xc051c8cb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc074f79980}}}, {0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}, 0xc073550280, 0xc051c8cb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc027ab17d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027ab17d0}, {0x40, 0xc074f79b70}, 0xc074f79ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc027ab17d0}, {0x4e9d7a0, 0xc0d7eebf00}, 0xc073550280, 0xc042fba720) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc027ab17d0}, 0xc02bc10540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc058d9aea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc058d9aea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802149 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc064b06c60}, 0xc03787f2f0, 0x1, 0xc0152b10a8, 0x519bbc, 0xc09efbede0, 0xc0494bb140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc064b06c60}, 0x120, 0xc0152b1230, 0x9efbed80, 0x0, 0x660069, 0xc09efbede0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc064b06c60}, 0xc09efbed80, {{{{0x0, 0x0}, {0xc03dcef8c0, 0x39}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc064b06c60}, 0x6352410, 0x351a9948) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc064b06c60}, {0xc09efbec60, {0x17430110173aa8b8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc064b06c60}, 0xc082adb4a0, 0xc0152b19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc064b06c60}, 0x660089, {0xc09efbec60, {0x17430110173aa8b8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc064b06c60}, 0xc06d00e800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110173aa8b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110173aa8b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064b06a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064b06a20}, {0xc0152b33f8, 0x660089}, 0xc0152b3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064b06a20}, {0xfec73c6679d80bea}, 0xc06d00e600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064b06990}, 0xc06d00e600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}, 0xc04ecdace0, 0xc0302f5950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}, 0x3, 0xc06e1c6c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0152b3980}}}, {0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}, 0xc04ecdace0, 0xc06e1c6c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc064b06990}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064b06990}, {0x40, 0xc0152b3b70}, 0xc0152b3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc064b06990}, {0x4e9d7a0, 0xc06d00e600}, 0xc04ecdace0, 0xc0302f5950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc064b06990}, 0xc04ad088a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0221b45a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0221b45a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804897 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc02fa0a8a0}, 0xc054314d68, 0x1, 0xc07b8bf0a8, 0x519bbc, 0xc0597f6240, 0xc057ce3500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02fa0a8a0}, 0x1e0, 0xc07b8bf230, 0x597f61e0, 0xc0103451e0, 0x18, 0xc0597f6240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc02fa0a8a0}, 0xc0597f61e0, {{{{0x0, 0x0}, {0xc057fd3200, 0x43}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc02fa0a8a0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc02fa0a8a0}, {0xc05a49c120, {0x174301103fe9505f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc02fa0a8a0}, 0xc07443e140, 0xc07b8bf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02fa0a8a0}, 0x660089, {0xc05a49c120, {0x174301103fe9505f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc02fa0a8a0}, 0xc057884c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103fe9505f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103fe9505f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02fa0a810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fa0a810}, {0xc07b8c13f8, 0x660089}, 0xc07b8c1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02fa0a810}, {0x7fe424ae2fb8}, 0xc057884a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02fa0a780}, 0xc057884a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}, 0xc0b85a7e60, 0xc054315710) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}, 0x3, 0xc029e1df40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b8c1980}}}, {0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}, 0xc0b85a7e60, 0xc029e1df40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02fa0a780}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fa0a780}, {0x40, 0xc07b8c1b70}, 0xc07b8c1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}, 0x20300a, 0x20300a) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02fa0a780}, {0x4e9d7a0, 0xc057884a00}, 0xc0b85a7e60, 0xc054315710) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02fa0a780}, 0xc02302ef60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05a49c000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05a49c000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800379 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc091518548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0915185b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc09171b1f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc09b117c00, {0xc002bec380, 0x2b, 0x34}, {0xc0766af0a0, 0x2b, 0x6d}, 0x1, 0xb1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc09171b088, {0xc002bec380, 0x1, 0x6d}, {0xc0766af0a0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09171b088, {0xc002bec380, 0x2b, 0x34}, {0xc0766af0a0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09171a7c0, {0xc0766af0a0, 0x6d, 0x34}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09171a7c0, {0xc002bec380, 0x2b, 0xc0766af0a0}, {0xc0766af0a0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09171a500, {0xc0766af0a0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0428f8008, {{0xc06dc2b4a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06ab87d00, {{0xc06dc2b4a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07439dc00, {0x63c85b8, 0xc0161cf1d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0161cf1d0}, {0x64a6560, 0xc06ab87d00}, {0xc06dc2b4a0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0161cf1d0}, {0x6466ed8, 0xc0428f8000}, {0xc06dc2b4a0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc026662990}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05b361300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020aba400, {0x63c85b8, 0xc0161cf1d0}, 0xc05b361300, 0xc048082d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020aba400, {0x63c85b8, 0xc0161cf1d0}, 0xc05b361300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0161cf140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0161cf140}, {0xc0032fe8f0, 0x660089}, 0xc0032fe8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0161cf140}, {0xc00102e4e0}, 0xc05b361200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0161cf0e0}, 0xc05b361200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b710}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803154 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06bc00, {0x63c85b8, 0xc04ff927b0}, 0xc04b2e3c00, 0xc07a98a460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc04ff927b0}, 0xc04b2e3c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e67b889, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e67b889, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ff92720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ff92720}, {0xc09ad506d8, 0x660089}, 0xc09ad506e0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ff92720}, {0xc00102e4e0}, 0xc04b2e3a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ff926c0}, 0xc04b2e3a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc024370630, {0x63c85b8, 0xc04ff92510}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc039a0f900, {0x63c85b8, 0xc04ff92510}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc039a0f900, {0x63c8510, 0xc01d8843c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc039a0f900, {0x63c8510, 0xc01d8843c0}, 0xc081e52cf0, {0x7fe42b790cf0, 0xc00e0d06c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc039a0f900, {0x63c85b8, 0xc021728d50}, {{0x63f88f8, 0xc081e52cf0}, {0xc08a1baa90, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc039a0f900, {0x63c85b8, 0xc021728d50}, {{0x63f88f8, 0xc081e52cf0}, {0xc08a1baa90, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc081e52cf0}, {0xc08a1baa90, 0x23}, 0x0, 0x0}, {0x139ab779, 0xedb7a834f, 0x0}, {0x139abcdf, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc039a0f900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc039a0f900, {0x63c8510, 0xc09a88b8c0}, 0xc076859dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc050a00190) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc09a88b8c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc050a00190) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219807586 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0b2c00, {0x63c85b8, 0xc045772870}, 0xc098245b00, 0xc0772d8e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc045772870}, 0xc098245b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110576c2c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110576c2c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0457727e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0457727e0}, {0xc0061c33f8, 0x660089}, 0xc0061c3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0457727e0}, {0x0}, 0xc098245900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045772720}, 0xc098245900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}, 0xc076f0d080, 0xc04ba29aa0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}, 0x3, 0xc035438180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0061c3980}}}, {0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}, 0xc076f0d080, 0xc035438180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045772720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045772720}, {0x40, 0xc0061c3b70}, 0xc0061c3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045772720}, {0x4e9d7a0, 0xc098245900}, 0xc076f0d080, 0xc04ba29aa0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045772720}, 0xc036dadd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08c1e5200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08c1e5200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811730 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc061dce000, {0x63c85b8, 0xc0204a5ec0}, 0xc071972100, 0xc075c62460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc0204a5ec0}, 0xc071972100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f6cccce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f6cccce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0204a5e00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0204a5e00}, {0xc0067c33f8, 0x660089}, 0xc0067c3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0204a5e00}, {0x7fe3ba666c98}, 0xc049b41e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0204a5d40}, 0xc049b41e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}, 0xc046d2a560, 0xc0b9f8d7d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}, 0x3, 0xc01b8b9000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0067c3980}}}, {0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}, 0xc046d2a560, 0xc01b8b9000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0204a5d40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0204a5d40}, {0x40, 0xc0067c3b70}, 0xc0067c3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0204a5d40}, {0x4e9d7a0, 0xc049b41e00}, 0xc046d2a560, 0xc0b9f8d7d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0204a5d40}, 0xc041cb8ea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03fa8e6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03fa8e6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800842 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc06fbb8450}, 0xc03a960720, 0x1, 0xc0482890a8, 0x519bbc, 0xc02d7a4420, 0xc028683140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06fbb8450}, 0x1e0, 0xc048289230, 0x2d7a43c0, 0x0, 0x660069, 0xc02d7a4420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc06fbb8450}, 0xc02d7a43c0, {{{{0x0, 0x0}, {0xc042ec0fc0, 0x53}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc0418474a0, {0x63c85b8, 0xc06fbb8450}, 0xc06ba32cf0, 0x48289428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc06fbb8450}, {0xc01ab0b680, {0x17430110001c2e22, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc06fbb8450}, 0xc0224f5a40, 0xc0482899f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06fbb8450}, 0x660089, {0xc01ab0b680, {0x17430110001c2e22, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc06fbb8450}, 0xc04dc25f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110001c2e22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110001c2e22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06fbb8360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fbb8360}, {0xc04828b3f8, 0x660089}, 0xc04828b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06fbb8360}, {0x666238362d613833}, 0xc04dc25d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fbb82d0}, 0xc04dc25d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}, 0xc019b26a20, 0xc0389756c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}, 0x3, 0xc01d6d2640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04828b980}}}, {0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}, 0xc019b26a20, 0xc01d6d2640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06fbb82d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fbb82d0}, {0x40, 0xc04828bb70}, 0xc04828bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06fbb82d0}, {0x4e9d7a0, 0xc04dc25d00}, 0xc019b26a20, 0xc0389756c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06fbb82d0}, 0xc0329544e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01ab0b560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01ab0b560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802204 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0777b3320}, 0xc0159bb050, 0x1, 0xc0627330a8, 0x519bbc, 0xc051109920, 0xc022510180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0777b3320}, 0x120, 0xc062733230, 0x511098c0, 0x0, 0x660069, 0xc051109920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0777b3320}, 0xc0511098c0, {{{{0x0, 0x0}, {0xc02957e480, 0xb6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0777b3320}, 0x6352410, 0x32cb3250) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0777b3320}, {0xc0511097a0, {0x17430110216405d4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0777b3320}, 0xc07839ca00, 0xc0627339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0777b3320}, 0x660089, {0xc0511097a0, {0x17430110216405d4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0777b3320}, 0xc019293100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110216405d4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110216405d4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0777b3290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0777b3290}, {0xc0627353f8, 0x660089}, 0xc062735400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0777b3290}, {0x6265396661373632}, 0xc019292f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0777b3200}, 0xc019292f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}, 0xc04098bae0, 0xc0756894b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}, 0x3, 0xc06c2d3000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062735980}}}, {0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}, 0xc04098bae0, 0xc06c2d3000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0777b3200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0777b3200}, {0x40, 0xc062735b70}, 0xc062735ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0777b3200}, {0x4e9d7a0, 0xc019292f00}, 0xc04098bae0, 0xc0756894b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0777b3200}, 0xc04c3f7e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc051109680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc051109680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803289 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc011fef400, {0x63c85b8, 0xc06e8c1500}, 0xc06eafce28, 0x1, 0xc04e1a90a8, 0x519bbc, 0xc0345e8ba0, 0xc033f8c2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06e8c1500}, 0x120, 0xc04e1a9230, 0x345e8b40, 0x0, 0x660069, 0xc0345e8ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc011fef400, {0x63c85b8, 0xc06e8c1500}, 0xc0345e8b40, {{{{0x0, 0x0}, {0xc05a482900, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06e8c1500}, 0x6352410, 0x2a838ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc008376140, {0x63c85b8, 0xc06e8c1500}, {0xc0345e8a20, {0x174301102ad31a60, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc008376140, {0x63c85b8, 0xc06e8c1500}, 0xc08863a320, 0xc04e1a99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06e8c1500}, 0x660089, {0xc0345e8a20, {0x174301102ad31a60, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000c5dc00, {0x63c85b8, 0xc06e8c1500}, 0xc07453e300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e8c1380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e8c1380}, {0xc04e1ab3f8, 0x660089}, 0xc04e1ab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e8c1380}, {0xc0015221e0}, 0xc07453e100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e8c12f0}, 0xc07453e100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}, 0xc06e7cc0a0, 0xc0592ba4f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}, 0x3, 0xc0211d77c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04e1ab980}}}, {0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}, 0xc06e7cc0a0, 0xc0211d77c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06e8c12f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e8c12f0}, {0x40, 0xc04e1abb70}, 0xc04e1abab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06e8c12f0}, {0x4e9d7a0, 0xc07453e100}, 0xc06e7cc0a0, 0xc0592ba4f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06e8c12f0}, 0xc01ea90720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0345e8900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0345e8900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803640 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc0592ed170}, 0xc02765e780, 0x1, 0xc0585bd0a8, 0x519bbc, 0xc047359320, 0xc04267c720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0592ed170}, 0x1e0, 0xc0585bd230, 0x473592c0, 0x0, 0x660069, 0xc047359320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc0592ed170}, 0xc0473592c0, {{{{0x0, 0x0}, {0xc07afbf320, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0592ed170}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc0592ed170}, {0xc045ea9c20, {0x174301102ef71c55, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc0592ed170}, 0xc022b45720, 0xc0585bd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0592ed170}, 0x660089, {0xc045ea9c20, {0x174301102ef71c55, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc0592ed170}, 0xc040e41b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ef71c55, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ef71c55, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592ed080}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592ed080}, {0xc0585bf3f8, 0x660089}, 0xc0585bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592ed080}, {0xc0015221e0}, 0xc040e41900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592ecff0}, 0xc040e41900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}, 0xc01ba25480, 0xc02d7ef4e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}, 0x3, 0xc039395680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0585bf980}}}, {0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}, 0xc01ba25480, 0xc039395680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0592ecff0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592ecff0}, {0x40, 0xc0585bfb70}, 0xc0585bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0592ecff0}, {0x4e9d7a0, 0xc040e41900}, 0xc01ba25480, 0xc02d7ef4e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0592ecff0}, 0xc065e22ea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc045ea9b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc045ea9b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810788 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0357c0, {0x63c85b8, 0xc07995a0f0}, 0xc0373e03a8, 0x1, 0xc023b190a8, 0x519bbc, 0xc04a5b7b60, 0xc074e0c3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07995a0f0}, 0x120, 0xc023b19230, 0x4a5b7b00, 0x0, 0x660069, 0xc04a5b7b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0357c0, {0x63c85b8, 0xc07995a0f0}, 0xc04a5b7b00, {{{{0x0, 0x0}, {0xc0637e6240, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07995a0f0}, 0x6352410, 0x55bf2460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd770, {0x63c85b8, 0xc07995a0f0}, {0xc04a5b79e0, {0x174301106d8bb05b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd770, {0x63c85b8, 0xc07995a0f0}, 0xc039b4b4a0, 0xc023b199f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07995a0f0}, 0x660089, {0xc04a5b79e0, {0x174301106d8bb05b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc07995a0f0}, 0xc086e92b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d8bb05b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d8bb05b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07995a060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07995a060}, {0xc023b1b3f8, 0x660089}, 0xc023b1b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07995a060}, {0x7fe13dc71e40}, 0xc086e92900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0438edef0}, 0xc086e92900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}, 0xc06734c2e0, 0xc0391cbfb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}, 0x3, 0xc015d26a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc023b1b980}}}, {0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}, 0xc06734c2e0, 0xc015d26a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0438edef0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0438edef0}, {0x40, 0xc023b1bb70}, 0xc023b1bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0438edef0}, {0x4e9d7a0, 0xc086e92900}, 0xc06734c2e0, 0xc0391cbfb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0438edef0}, 0xc03090f8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04a5b78c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04a5b78c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219663655 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3dfc80, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0557c8500, 0xc058158800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0557c8500, {0xc058158800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0557c8500, {0xc058158800, 0x735, 0xc05fe36dc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc025d61c00, {0xc058158800, 0x7eda79, 0xc058158805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc016ce9ce0, {0xc058158800, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b9f8d6e0, {0xc058158800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc048b925f8, {0x631b900, 0xc0b9f8d6e0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc048b92380, {0x7fe42ab59310, 0xc016ce9ce0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc048b92380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc048b92380, {0xc082ec0000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02620d230, {0xc082ec0000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc01278a088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc01278a088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc01278a660, {0x634aac0, 0xc01278a088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc01278a000, 0xc076bb32d0, 0xc086a3742b, 0xc086a3742d, {0x635bec0, 0xc0144335f0}, 0xc000d6ae00, 0xc02a515d28, 0xc086a37429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc01278a000, {0x63c85b8, 0xc044d85770}, 0xc076bb32c0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a0485038, 0x8}}, 0x0, 0xc044d85710, 0xc044d85740, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc044d85770}, {0x643d810, 0xc016ce9ce0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06e70d0e0}, {0x643d810, 0xc016ce9ce0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06e70d0e0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809989 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bc672c0, {0x63c85b8, 0xc076ed9920}, 0xc02febede0, 0x1, 0x70801805d9ab36, 0x37e11d600, 0xc07aafa840, 0xc082a3a720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc076ed9920}, 0x120, 0xc028321230, 0x7aafa7e0, 0x0, 0x660069, 0xc07aafa840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bc672c0, {0x63c85b8, 0xc076ed9920}, 0xc07aafa7e0, {{{{0x0, 0x0}, {0xc073078d80, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc076ed9920}, 0x6352410, 0x43a28758) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bc68910, {0x63c85b8, 0xc076ed9920}, {0xc07aafa6c0, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bc68910, {0x63c85b8, 0xc076ed9920}, 0xc04cf17a40, 0xc0283219f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc076ed9920}, 0x660089, {0xc07aafa6c0, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bf94800, {0x63c85b8, 0xc076ed9920}, 0xc08b934f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076ed9890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076ed9890}, {0xc0283233f8, 0x660089}, 0xc028323400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076ed9890}, {0x0}, 0xc08b934d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076ed9800}, 0xc08b934d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}, 0xc01e9ee2a0, 0xc02febedb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}, 0x3, 0xc0412e5940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028323980}}}, {0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}, 0xc01e9ee2a0, 0xc0412e5940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076ed9800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076ed9800}, {0x40, 0xc028323b70}, 0xc028323ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076ed9800}, {0x4e9d7a0, 0xc08b934d00}, 0xc01e9ee2a0, 0xc02febedb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076ed9800}, 0xc052799e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c5c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c5c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808290 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bbe0280, {0x63c85b8, 0xc037323470}, 0xc037bcb698, 0x1, 0x0, 0x1, 0xc07cab8600, 0xc07854ca80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc037323470}, 0x120, 0xc00eb3c728, 0x7cab85a0, 0x0, 0x660069, 0xc07cab8600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bbe0280, {0x63c85b8, 0xc037323470}, 0xc07cab85a0, {{{{0x0, 0x0}, {0xc060492c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc037323470}, 0x6352410, 0x2ad6f788) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bae13b0, {0x63c85b8, 0xc037323470}, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bae13b0, {0x63c85b8, 0xc037323470}, 0xc052005b80, 0xc07c6c4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc037323470}, 0x660089, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bbe6c00, {0x63c85b8, 0xc037323470}, 0xc01fb36900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0373233e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0373233e0}, {0xc0936368f0, 0x660089}, 0xc0936368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0373233e0}, {0xc00102e4e0}, 0xc01fb36800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc037323380}, 0xc01fb36800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0373226f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803363 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04df30840}, 0xc02990d1b8, 0x1, 0xc06094d0a8, 0x519bbc, 0xc044dced80, 0xc0420c29c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04df30840}, 0x1e0, 0xc06094d230, 0x44dced20, 0x0, 0x660069, 0xc044dced80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04df30840}, 0xc044dced20, {{{{0x0, 0x0}, {0xc077c91b00, 0x3c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04df30840}, 0x63521a0, 0x41bc4c18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04df30840}, {0xc0332dde60, {0x174301102b2a2ba3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04df30840}, 0xc05c7025a0, 0xc06094d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04df30840}, 0x660089, {0xc0332dde60, {0x174301102b2a2ba3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04df30840}, 0xc0b73d7500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b2a2ba3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b2a2ba3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04df307b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04df307b0}, {0xc06094f3f8, 0x660089}, 0xc06094f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04df307b0}, {0x54414450555f5353}, 0xc0b73d7300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04df30720}, 0xc0b73d7300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}, 0xc04618f4c0, 0xc027025bd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}, 0x3, 0xc0835f7000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06094f980}}}, {0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}, 0xc04618f4c0, 0xc0835f7000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04df30720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04df30720}, {0x40, 0xc06094fb70}, 0xc06094fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04df30720}, {0x4e9d7a0, 0xc0b73d7300}, 0xc04618f4c0, 0xc027025bd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04df30720}, 0xc024d41c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0332ddd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0332ddd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 202012964 [select, 31 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0606adb00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc03e655550, {0x4cfe6e0, 0xc043d30dc0}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0606ad8c0, 0xc0072d95b0, 0xc0072d9580) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0606ad8c0, {0x4cfe6e0, 0xc043d30dc0}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc06145bb60, {0x4cfe6e0, 0xc043d30dc0}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc030a51440) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc030a51440}, 0xc019327440, 0xc000cb3540, {0x63c85b8, 0xc037a8f8f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc037a8f8f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 218871513 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc02800a550, 0x71) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc074caf500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc074cafb60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05a8ff300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05a8ff300, {0x63c8510, 0xc02800a580}, 0xc08372fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04fefed10) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc02800a580}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04fefed10) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802003 [semacquire]: sync.runtime_SemacquireMutex(0x5b5dbf2, 0x80, 0xc09278c3a8) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc000088000, 0xc09278c410, 0x49e6b9) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x119405e0902107f, 0x300, 0xc, 0x9baf000) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7236e27, 0x3b81}, 0x0, 0xc02331c278) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc02331c000, {0xffffffffffffff15, 0xc09278c5b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc02331c000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc02331c000, {0xc0361a6c80, 0x32, 0x74}, 0x13, 0x15) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc02331c000, {0xc00bfdca50, 0x32, 0x74}, {0xc0361a6c80, 0x51c9860, 0xc0361a6c80}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc02331c000, {0xc00bfdca50, 0x32, 0xa6}, {0xc0361a6c80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b2118898, {0xc00bfdca50, 0x32, 0xa6}, {0xc0361a6c80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b21171c0, {0xc0361a6c80, 0x74, 0xa6}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b21171c0, {0xc00bfdca50, 0x74, 0xc0361a6c80}, {0xc0361a6c80, 0xc0216a32c0, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b2116f00, {0xc0361a6c80, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc055f579a8, {{0xc0216a32c0, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc059f4de00, {{0xc09be3e180, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07dbea000, {0x63c85b8, 0xc05a5daa20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc05a5daa20}, {0x64a6560, 0xc059f4de00}, {0xc09be3e180, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc05a5daa20}, {0x6466ed8, 0xc055f57700}, {0xc09be3e180, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0180739b0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc076191100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020eb9800, {0x63c85b8, 0xc05a5daa20}, 0xc076191100, 0xc089725ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020eb9800, {0x63c85b8, 0xc05a5daa20}, 0xc076191100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05a5da990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05a5da990}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05a5da990}, {0xc00102e4e0}, 0xc076191000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05a5da930}, 0xc076191000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618cb1d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809529 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc0720f5080}, 0xc02765fb78, 0x1, 0xc01c0c50a8, 0x519bbc, 0xc06d0abb60, 0xc04fa34cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0720f5080}, 0x120, 0xc01c0c5230, 0x6d0abb00, 0x0, 0x660069, 0xc06d0abb60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc0720f5080}, 0xc06d0abb00, {{{{0x0, 0x0}, {0xc04b734c60, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0720f5080}, 0x6352410, 0x603b9250) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc0720f5080}, {0xc06d0ab9e0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc0720f5080}, 0xc04c6ff5e0, 0xc01c0c59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0720f5080}, 0x660089, {0xc06d0ab9e0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc0720f5080}, 0xc04fe3da00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0720f4f90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0720f4f90}, {0xc01c0c73f8, 0x660089}, 0xc01c0c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0720f4f90}, {0xaa201743011002b3}, 0xc04fe3d700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0720f4ed0}, 0xc04fe3d700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}, 0xc07c1843e0, 0xc0430a41f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}, 0x3, 0xc06157a1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01c0c7980}}}, {0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}, 0xc07c1843e0, 0xc06157a1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0720f4ed0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0720f4ed0}, {0x40, 0xc01c0c7b70}, 0xc01c0c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0720f4ed0}, {0x4e9d7a0, 0xc04fe3d700}, 0xc07c1843e0, 0xc0430a41f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0720f4ed0}, 0xc074f9b740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086f69440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086f69440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810894 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc01e71a3f0}, 0xc01f9b6e28, 0x1, 0xc0087e10a8, 0x519bbc, 0xc0645d9140, 0xc0428b7140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc01e71a3f0}, 0x1e0, 0xc0087e1230, 0x645d90e0, 0x0, 0x660069, 0xc0645d9140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc01e71a3f0}, 0xc0645d90e0, {{{{0x0, 0x0}, {0xc042835440, 0xd}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc01e71a3f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc01e71a3f0}, {0xc022d21d40, {0x174301106fac0bd5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc01e71a3f0}, 0xc071f02d20, 0xc0087e19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01e71a3f0}, 0x660089, {0xc022d21d40, {0x174301106fac0bd5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc01e71a3f0}, 0xc049113b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106fac0bd5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106fac0bd5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e71a360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e71a360}, {0xc0087e33f8, 0x660089}, 0xc0087e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e71a360}, {0x7fdd4a38b888}, 0xc049113900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e71a2d0}, 0xc049113900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}, 0xc072222ea0, 0xc0b27aa840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}, 0x3, 0xc04fe1ca00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0087e3980}}}, {0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}, 0xc072222ea0, 0xc04fe1ca00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01e71a2d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e71a2d0}, {0x40, 0xc0087e3b70}, 0xc0087e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01e71a2d0}, {0x4e9d7a0, 0xc049113900}, 0xc072222ea0, 0xc0b27aa840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01e71a2d0}, 0xc047033740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc022d21b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc022d21b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803222 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc05934b770}, 0xc01227abb8, 0x1, 0xc0617d50a8, 0x519bbc, 0xc034e76720, 0xc033fd0f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05934b770}, 0x120, 0xc0617d5230, 0x34e766c0, 0x0, 0x660069, 0xc034e76720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc05934b770}, 0xc034e766c0, {{{{0x0, 0x0}, {0xc073d950e0, 0x13}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05934b770}, 0x6352410, 0x3ad19fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc05934b770}, {0xc034e765a0, {0x174301102ad31a60, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc05934b770}, 0xc0720b0f00, 0xc0617d59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05934b770}, 0x660089, {0xc034e765a0, {0x174301102ad31a60, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc05934b770}, 0xc0517d4200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05934b6e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05934b6e0}, {0xc0617d73f8, 0x660089}, 0xc0617d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05934b6e0}, {0x7fe13e8e5a10}, 0xc0517d4000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05934b650}, 0xc0517d4000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}, 0xc02095b5a0, 0xc06c0e0c18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}, 0x3, 0xc04375fa40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0617d7980}}}, {0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}, 0xc02095b5a0, 0xc04375fa40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05934b650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05934b650}, {0x40, 0xc0617d7b70}, 0xc0617d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05934b650}, {0x4e9d7a0, 0xc0517d4000}, 0xc02095b5a0, 0xc06c0e0c18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05934b650}, 0xc01a22f200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc033fcc240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc033fcc240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812458 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc052b3a1b0}, 0xc09dc1b188, 0x1, 0xc03a6550a8, 0x519bbc, 0xc0a95143c0, 0xc0c3e2acc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc052b3a1b0}, 0x120, 0xc03a655230, 0xa9514360, 0x0, 0x660069, 0xc0a95143c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc052b3a1b0}, 0xc0a9514360, {{{{0x0, 0x0}, {0xc093c9bd40, 0x121}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc052b3a1b0}, 0x6352410, 0x42012f80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc052b3a1b0}, {0xc0a9514240, {0x17430110802c6749, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc052b3a1b0}, 0xc064dfe1e0, 0xc03a6559f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc052b3a1b0}, 0x660089, {0xc0a9514240, {0x17430110802c6749, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc052b3a1b0}, 0xc0435ddc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110802c6749, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110802c6749, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc052b3a120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052b3a120}, {0xc03a6573f8, 0x660089}, 0xc03a657400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc052b3a120}, {0xc0015221e0}, 0xc0435dda00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc052b3a090}, 0xc0435dda00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}, 0xc0adac66e0, 0xc04434edf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}, 0x3, 0xc06051a3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03a657980}}}, {0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}, 0xc0adac66e0, 0xc06051a3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc052b3a090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052b3a090}, {0x40, 0xc03a657b70}, 0xc03a657ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc052b3a090}, {0x4e9d7a0, 0xc0435dda00}, 0xc0adac66e0, 0xc04434edf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc052b3a090}, 0xc0369b2540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0a9514120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0a9514120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 204400954 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1800, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0675e4c00, 0xc084ac7000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0675e4c00, {0xc084ac7000, 0x73d, 0x73d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0675e4c00, {0xc084ac7000, 0x5, 0xc08da9cbc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01f3b2f28, {0xc084ac7000, 0x40, 0x6}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02955bd40, {0xc084ac7000, 0x461247, 0x73d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc041138408, {0xc084ac7000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04563e978, {0x631b900, 0xc041138408}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04563e700, {0x7fe42ab59310, 0xc02955bd40}, 0x73d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04563e700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04563e700, {0xc0a96f6000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0293d93b0, {0xc0a96f6000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc038a3e088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc038a3e088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc038a3e660, {0x634aac0, 0xc038a3e088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc038a3e000, 0xc01ae815c0, 0xc08da9d42b, 0xc08da9d42d, {0x635bec0, 0xc01dc94000}, 0xc000d6ae00, 0xc026766458, 0xc08da9d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc038a3e000, {0x63c85b8, 0xc021f3bd40}, 0xc01ae815b0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a93ec033, 0x10}}, 0x0, 0xc021f3bc50, 0xc021f3bd10, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc021f3bd40}, {0x643d810, 0xc02955bd40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc041fb6c30}, {0x643d810, 0xc02955bd40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc041fb6c30}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810136 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02fe28c60}, 0xc088190cb0, {0x646cc48, 0xc02fe28d80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02fe28c60}, 0xedb7a8350, 0xc0165b27e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02fe28c60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc076482990}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc088190ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc076482990}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 203350914 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a59b80, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06f148280, 0xc0481db000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06f148280, {0xc0481db000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06f148280, {0xc0481db000, 0x735, 0xc05687b400}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc033b74320, {0xc0481db000, 0x7eda79, 0xc0481db005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06cfae5a0, {0xc0481db000, 0xc079fe4c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc05bebf6f8, {0xc0481db000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06f4493f8, {0x631b900, 0xc05bebf6f8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06f449180, {0x7fe42ab59310, 0xc06cfae5a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06f449180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06f449180, {0xc024e0e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01386d1a0, {0xc024e0e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03281d588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03281d588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03281db60, {0x634aac0, 0xc03281d588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03281d500, 0xc0795a0da0, 0xc082ad742b, 0xc082ad742d, {0x635bec0, 0xc02a5fbef0}, 0xc000d6ae00, 0xc030b54588, 0xc082ad7429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03281d500, {0x63c85b8, 0xc057213dd0}, 0xc0795a0d90, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc04cbe4025, 0x8}}, 0x0, 0xc057213d70, 0xc057213da0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc057213dd0}, {0x643d810, 0xc06cfae5a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc05c417b90}, {0x643d810, 0xc06cfae5a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc05c417b90}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808662 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0310c8280, {0x63c85b8, 0xc01b6934a0}, 0xc0442eac18, 0x1, 0xc00d7d50a8, 0x519bbc, 0xc084421140, 0xc09a1e76e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01b6934a0}, 0x120, 0xc00d7d5230, 0x844210e0, 0x0, 0x660069, 0xc084421140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0310c8280, {0x63c85b8, 0xc01b6934a0}, 0xc0844210e0, {{{{0x0, 0x0}, {0xc0639cea20, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01b6934a0}, 0x6352410, 0x321a1ef0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc03107e910, {0x63c85b8, 0xc01b6934a0}, {0xc084420fc0, {0x1743011055697687, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc03107e910, {0x63c85b8, 0xc01b6934a0}, 0xc076578140, 0xc00d7d59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01b6934a0}, 0x660089, {0xc084420fc0, {0x1743011055697687, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310bf000, {0x63c85b8, 0xc01b6934a0}, 0xc0a2055c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011055697687, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011055697687, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b6933e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b6933e0}, {0xc00d7d73f8, 0x660089}, 0xc00d7d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b6933e0}, {0x4031fd613255633f}, 0xc0a2055a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b693320}, 0xc0a2055a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}, 0xc034e6cd40, 0xc03c1d4600) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}, 0x3, 0xc055a9ce00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00d7d7980}}}, {0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}, 0xc034e6cd40, 0xc055a9ce00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01b693320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b693320}, {0x40, 0xc00d7d7b70}, 0xc00d7d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01b693320}, {0x4e9d7a0, 0xc0a2055a00}, 0xc034e6cd40, 0xc03c1d4600) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01b693320}, 0xc05576a960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc084420d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc084420d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810348 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0211a7950}, 0xc015224c90, 0x1, 0xc02839d0a8, 0x519bbc, 0xc040039aa0, 0xc065e01aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0211a7950}, 0x1e0, 0xc02839d230, 0x40039a40, 0x0, 0x660069, 0xc040039aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0211a7950}, 0xc040039a40, {{{{0x0, 0x0}, {0xc0bc5b9200, 0xfd}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc072596840, {0x63c85b8, 0xc0211a7950}, 0xc057944e50, 0x2839d428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0211a7950}, {0xc047529320, {0x17430110678b3252, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0211a7950}, 0xc07cc6fcc0, 0xc02839d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0211a7950}, 0x660089, {0xc047529320, {0x17430110678b3252, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0211a7950}, 0xc075382f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110678b3252, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110678b3252, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0211a7560}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0211a7560}, {0xc02839f3f8, 0x660089}, 0xc02839f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0211a7560}, {0x7fe3b5948f48}, 0xc075382d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0211a74d0}, 0xc075382d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}, 0xc08bdf31e0, 0xc063fa6138) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}, 0x3, 0xc0689ecfc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02839f980}}}, {0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}, 0xc08bdf31e0, 0xc0689ecfc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0211a74d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0211a74d0}, {0x40, 0xc02839fb70}, 0xc02839fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0211a74d0}, {0x4e9d7a0, 0xc075382d00}, 0xc08bdf31e0, 0xc063fa6138) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0211a74d0}, 0xc0be750f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc047529200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc047529200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 205581621 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b3030, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc077707480, 0xc00da3f800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc077707480, {0xc00da3f800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc077707480, {0xc00da3f800, 0x0, 0xc081ec4bc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc049d92bb0, {0xc00da3f800, 0x100, 0x6}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0166cc900, {0xc00da3f800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc041138450, {0xc00da3f800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc001574278, {0x631b900, 0xc041138450}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc001574000, {0x7fe42ab59310, 0xc0166cc900}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc001574000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc001574000, {0xc013968000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03077d8f0, {0xc013968000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a1a7cb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a1a7cb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a1a7d0e0, {0x634aac0, 0xc0a1a7cb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a1a7ca80, 0xc0297c02b0, 0xc08714d42b, 0xc08714d42d, {0x635bec0, 0xc01d7b7ef0}, 0xc000d6ae00, 0xc0470b0508, 0xc08714d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a1a7ca80, {0x63c85b8, 0xc04c6f6330}, 0xc0297c02a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc013954025, 0x8}}, 0x0, 0xc04c6f62d0, 0xc04c6f6300, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04c6f6330}, {0x643d810, 0xc0166cc900}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02d3ac6f0}, {0x643d810, 0xc0166cc900}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02d3ac6f0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802720 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc02211ef60}, 0xc0569ad518, 0x1, 0xc064f310a8, 0x519bbc, 0xc04934e4e0, 0xc03d1f1da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02211ef60}, 0x120, 0xc064f31230, 0x4934e480, 0x0, 0x660069, 0xc04934e4e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc02211ef60}, 0xc04934e480, {{{{0x0, 0x0}, {0xc0b72f18c0, 0x12}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02211ef60}, 0x6352410, 0x1a06fde0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc02211ef60}, {0xc04934e360, {0x1743011024123859, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc02211ef60}, 0xc082b2a820, 0xc064f319f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02211ef60}, 0x660089, {0xc04934e360, {0x1743011024123859, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc02211ef60}, 0xc0aebeac00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024123859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024123859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02211ed80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02211ed80}, {0xc064f333f8, 0x660089}, 0xc064f33400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02211ed80}, {0x7fe3b7a80c30}, 0xc0aebeaa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02211ecf0}, 0xc0aebeaa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}, 0xc086683f00, 0xc01e0e1440) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}, 0x3, 0xc036e3d300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc064f33980}}}, {0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}, 0xc086683f00, 0xc036e3d300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02211ecf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02211ecf0}, {0x40, 0xc064f33b70}, 0xc064f33ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02211ecf0}, {0x4e9d7a0, 0xc0aebeaa00}, 0xc086683f00, 0xc01e0e1440) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02211ecf0}, 0xc01c825620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc046612360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc046612360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807778 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc07c061bf0}, 0xc06c0e0540, 0x1, 0xc0036e30a8, 0x519bbc, 0xc086627500, 0xc08b529aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07c061bf0}, 0x1e0, 0xc0036e3230, 0x866274a0, 0x0, 0x660069, 0xc086627500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc07c061bf0}, 0xc0866274a0, {{{{0x0, 0x0}, {0xc0566a8480, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07c061bf0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc07c061bf0}, {0xc053ff5d40, {0x1743011058d53dfd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc07c061bf0}, 0xc02d63c280, 0xc0036e39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07c061bf0}, 0x660089, {0xc053ff5d40, {0x1743011058d53dfd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc07c061bf0}, 0xc06212f200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058d53dfd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058d53dfd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c061b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c061b60}, {0xc0036e53f8, 0x660089}, 0xc0036e5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c061b60}, {0x3feffffdd78f7de3}, 0xc06212f000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c061ad0}, 0xc06212f000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}, 0xc0d55ba4e0, 0xc03bb18f78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}, 0x3, 0xc0141e5a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0036e5980}}}, {0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}, 0xc0d55ba4e0, 0xc0141e5a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c061ad0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c061ad0}, {0x40, 0xc0036e5b70}, 0xc0036e5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c061ad0}, {0x4e9d7a0, 0xc06212f000}, 0xc0d55ba4e0, 0xc03bb18f78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c061ad0}, 0xc07a998120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc053ff5c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc053ff5c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219769707 [runnable]: github.com/cockroachdb/cockroach/pkg/sql/opt/memo.(*ScanExpr).Relational(0xc03a7621f8) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/opt/memo/expr.og.go:1464 +0x3c github.com/cockroachdb/cockroach/pkg/sql/opt/memo.(*SortExpr).Relational(0xc03ebe23a0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/opt/memo/expr.og.go:332 +0x22 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*coster).ComputeCost(0xc03ebe23a0, {0x648cc10, 0xc03112d6e0}, 0x9b977e0) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/coster.go:543 +0x7e7 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeEnforcer(0xc03ebe22d8, 0xc04d154348, {0x648cc10, 0xc03112d6e0}, 0xc06ffe87e0, {0x648c8b0, 0xc03a7621f8}, 0xc095b779b8) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:658 +0x8e github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).enforceProps(0xc03ebe22d8, 0xc04d154348, {0x648c8b0, 0xc03a7621f8}, 0xc06ffe87e0) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:619 +0xfe github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroupMember(0xc03ebe22d8, 0xc04d154348, {0x648c8b0, 0xc03a7621f8}, 0xc06ffe87e0) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:522 +0x45 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroup(0xc03ebe22d8, {0x648c8b0, 0xc03a7621f8}, 0xc06ffe87e0) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:482 +0x366 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeExpr(0xc03ebe22d8, {0x63f4a78, 0xc03a7621f8}, 0xc06ffe8690) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:265 +0x11e github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroupMember(0xc03ebe22d8, 0xc04d1540f0, {0x648c940, 0xc0501186f8}, 0xc06ffe8690) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:537 +0x105 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroup(0xc03ebe22d8, {0x648c940, 0xc0501186f8}, 0xc06ffe8690) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:482 +0x366 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeExpr(0xc03ebe22d8, {0x63f4ab8, 0xc0501186f8}, 0x9b977e0) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:265 +0x11e github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroupMember(0xc03ebe22d8, 0xc04d154078, {0x648be90, 0xc050118978}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:537 +0x105 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroup(0xc03ebe22d8, {0x648be90, 0xc050118978}, 0x9b977e0) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:482 +0x366 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeEnforcer(0xc03ebe22d8, 0xc04d154000, {0x648b350, 0xc032e28420}, 0xc06ffe8620, {0x648be90, 0xc050118978}, 0x120) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:653 +0x54 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).enforceProps(0xc03ebe22d8, 0xc04d154000, {0x648be90, 0xc050118978}, 0xc06ffe8620) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:612 +0x2cd github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroupMember(0xc03ebe22d8, 0xc04d154000, {0x648be90, 0xc050118978}, 0x1f244c8) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:522 +0x45 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroup(0xc03ebe22d8, {0x648be90, 0xc050118978}, 0xc06ffe8620) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:482 +0x366 github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).Optimize(0xc03ebe22d8) github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:235 +0x132 github.com/cockroachdb/cockroach/pkg/sql.(*optPlanningCtx).reuseMemo(0xc03ebe2280, 0xc058825860) github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:484 +0x85 github.com/cockroachdb/cockroach/pkg/sql.(*optPlanningCtx).buildExecMemo(0xc03ebe2280, {0x63c8510, 0xc012b20500}) github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:514 +0x17e github.com/cockroachdb/cockroach/pkg/sql.(*planner).makeOptimizerPlan(0xc03ebe19f0, {0x63c8510, 0xc012b20500}) github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:226 +0xff github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).makeExecPlan(0xc03ebe1300, {0x63c8510, 0xc012b20500}, 0xc03ebe19f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1433 +0x4e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc03ebe1300, {0x63c8510, 0xc012b20500}, 0xc03ebe19f0, {0x7fe42b790cf0, 0xc0577dbbc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1103 +0x24e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc03ebe1300, {0x63c85b8, 0xc06c833b60}, {{0x63f8d78, 0xc03e8ccbe0}, {0xc0826aa838, 0x185}, 0x1, 0x1}, 0xc03d0cc5a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc03ebe1300, {0x63c85b8, 0xc06c833b60}, {{0x63f8d78, 0xc03e8ccbe0}, {0xc0826aa838, 0x185}, 0x1, 0x1}, 0xc03d0cc5a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc03ebe1300, {0x63c85b8, 0xc06c833b60}, {0xc03d0cc5a0, {0xc02259cac0, 0x1, 0x1}, {0xc03bc2d980, 0x10, 0x10}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0826aaa36, 0x0}, 0x0, {0xc872378, 0xedb7a8351, 0x0}, 0x1}, 0xc03ebe1300, 0xc095b79980, 0xc095b79970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03ebe1300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03ebe1300, {0x63c8510, 0xc044871dc0}, 0xc058167dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02bb2ef40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc044871dc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02bb2ef40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812537 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074aa90, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00d1618c0, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc084a94000, 0xc00d1618c0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00d1618c0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a523c0, 0xc0339ef198, 0xc090e2ba38, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc090e2bb50, 0x0, 0xc090e2bb50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc090e2b9e8, 0xc0158cbe40, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc090e2b9e8, {0xc030260b00, 0x28, 0x73}, {0xc02c1baeb0, 0x28, 0x4b}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc090e2a7c0, {0xc02c1baeb0, 0x0, 0xc0339ef4e0}, 0x121ea30, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc090e2a7c0, {0xc030260b00, 0x28, 0x4b}, {0xc02c1baeb0, 0x27, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc090e2a500, {0xc02c1baeb0, 0x28, 0x4b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc07d86fb88, {{0xc03a992630, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04c2ddc00, {{0xc03a992630, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc082682800, {0x63c85b8, 0xc04bf648d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04bf648d0}, {0x64a6560, 0xc04c2ddc00}, {0xc03a992630, 0xc00bc1fa40, 0x30}, {0x1743011086e17ffd, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04bf648d0}, {0x6466ed8, 0xc07d86fb80}, {0xc03a992630, 0x27, 0x30}, {0x1743011086e17ffd, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0257d8fa8}, {{0x1743011086e17ffd, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011086e17ffd, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04aa84800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a4aee4fd, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02f355000, {0x63c85b8, 0xc04bf648d0}, 0xc04aa84800, 0xc089013c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f355000, {0x63c85b8, 0xc04bf648d0}, 0xc04aa84800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011086e17ffd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011086e17ffd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bf64840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf64840}, {0xc0339f53f8, 0x660089}, 0xc0339f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bf64840}, {0x7fde180393b8}, 0xc04aa84600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bf647b0}, 0xc04aa84600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}, 0xc0ccf7f5e0, 0xc038161470) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}, 0x3, 0xc02a52e580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0339f5980}}}, {0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}, 0xc0ccf7f5e0, 0xc02a52e580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bf647b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf647b0}, {0x40, 0xc0339f5b70}, 0xc0339f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bf647b0}, {0x4e9d7a0, 0xc04aa84600}, 0xc0ccf7f5e0, 0xc038161470) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bf647b0}, 0xc05c2fa4e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdd680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdd680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806215 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc070c9d860}, 0xc08d16a138, 0x1, 0x0, 0x1, 0xc044ddcde0, 0xc04eb8d800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc070c9d860}, 0x120, 0xc008078728, 0x44ddcd80, 0x0, 0x660069, 0xc044ddcde0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc070c9d860}, 0xc044ddcd80, {{{{0x0, 0x0}, {0xc08f9d2d80, 0x22}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc070c9d860}, 0x6352410, 0x24a007a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc070c9d860}, {0xc044ddcc60, {0x174301104c82f06f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc070c9d860}, 0xc030615180, 0xc06f204ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc070c9d860}, 0x660089, {0xc044ddcc60, {0x174301104c82f06f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc070c9d860}, 0xc065023300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c82f06f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c82f06f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070c9d7d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070c9d7d0}, {0xc09363c8f0, 0x660089}, 0xc09363c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070c9d7d0}, {0xc00102e4e0}, 0xc065023200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070c9d770}, 0xc065023200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc070c9d590}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800575 [runnable]: sync.runtime_SemacquireMutex(0xc05b228690, 0x8f, 0x7fe3c8121f00) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc05b228538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc05b228560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc027690480, {0xffffffffffffff15, 0xc05b2286c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc027690480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc027690480, {0xc04655d200, 0x32, 0x90}, {0xc02e0cfd80, 0x32, 0x7e}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093393998, {0xc04655d200, 0x32, 0x90}, {0xc02e0cfd80, 0x32, 0x7e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0933922c0, {0xc02e0cfd80, 0x7e, 0x90}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0933922c0, {0xc04655d200, 0x7e, 0xc02e0cfd80}, {0xc02e0cfd80, 0xc00031e680, 0x31}, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093392000, {0xc02e0cfd80, 0x32, 0x7e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04ded0ea8, {{0xc00031e680, 0x31, 0xd0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc047ba4a00, {{0xc0a423a750, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04a8d7c00, {0x63c85b8, 0xc01f532810}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01f532810}, {0x64a6560, 0xc047ba4a00}, {0xc0a423a750, 0x10932519aa2c0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01f532810}, {0x6466ed8, 0xc04ded0c00}, {0xc0a423a750, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01b19fa58}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0546a4f00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b11000, {0x63c85b8, 0xc01f532810}, 0xc0546a4f00, 0xc06477dcc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b11000, {0x63c85b8, 0xc01f532810}, 0xc0546a4f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01f532720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01f532720}, {0xc07d2528f0, 0x660089}, 0xc07d2528f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01f532720}, {0xc00102e4e0}, 0xc0546a4e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01f5326c0}, 0xc0546a4e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc021185f20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791854 [runnable]: syscall.Syscall6(0x11, 0x4675, 0x7fdf30b11020, 0x2885, 0x4bc2e6, 0x0, 0x0) GOROOT/src/syscall/asm_linux_amd64.s:43 +0x5 syscall.Pread(0xc08d9ee2e0, {0x7fdf30b11020, 0x9bb0c40, 0xc00074ae10}, 0x0) GOROOT/src/syscall/zsyscall_linux_amd64.go:1212 +0x58 internal/poll.(*FD).Pread(0xc01d259380, {0x7fdf30b11020, 0x4ba25b, 0x2885}, 0x200000003) GOROOT/src/internal/poll/fd_unix.go:193 +0xf9 os.(*File).pread(...) GOROOT/src/os/file_posix.go:41 os.(*File).ReadAt(0xc032cb2620, {0x7fdf30b11020, 0xc08d9ee3c0, 0x1184134}, 0x28a5) GOROOT/src/os/file.go:137 +0x191 github.com/cockroachdb/pebble/vfs.enospcFile.ReadAt(...) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_full.go:361 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc059ab38c0, {0x4bc2e6, 0x2880}, 0x0, 0xc0842f6b78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2339 +0x3f2 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0842f6900, {0xffffffffffffff15, 0xc08d9ee5c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0842f6900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0842f6900, {0xc04c2bd110, 0x2b, 0x6c}, 0x13, 0xad) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0842f6900, {0xc02bef4280, 0x2b, 0x6c}, {0xc04c2bd110, 0x51c9860, 0xc04c2bd110}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0842f6900, {0xc02bef4280, 0x2b, 0x74}, {0xc04c2bd110, 0x2b, 0x6c}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc085a4a640, {0xc02bef4280, 0x2b, 0x74}, {0xc04c2bd110, 0x2b, 0x6c}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc085a491c0, {0xc04c2bd110, 0x6c, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc085a491c0, {0xc02bef4280, 0x2b, 0xc04c2bd110}, {0xc04c2bd110, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc085a48f00, {0xc04c2bd110, 0x2b, 0x6c}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04a2ac008, {{0xc05ad7d590, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc08256b900, {{0xc05ad7d590, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc088a02c00, {0x63c85b8, 0xc074d666f0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc074d666f0}, {0x64a6560, 0xc08256b900}, {0xc05ad7d590, 0x14, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc074d666f0}, {0x6466ed8, 0xc04a2ac000}, {0xc05ad7d590, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc057246c30}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06d618d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f82f400, {0x63c85b8, 0xc074d666f0}, 0xc06d618d00, 0xc06891ad20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f82f400, {0x63c85b8, 0xc074d666f0}, 0xc06d618d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074d66660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074d66660}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074d66660}, {0xc00102e4e0}, 0xc06d618c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074d66600}, 0xc06d618c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc053ec6a80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809856 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01dd05000, {0x63c85b8, 0xc061235860}, 0xc062d7b500, 0xc0787280a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc061235860}, 0xc062d7b500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bff536f, 0x0, 0x0}, 0xc03cc76890, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bff536f, 0x0, 0x0}, 0xc03cc76890, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0612357d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0612357d0}, {0xc00df473f8, 0x660089}, 0xc00df47400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0612357d0}, {0x7fe429311c20}, 0xc062d7b300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc061235740}, 0xc062d7b300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}, 0xc02be17c40, 0xc041bf3938) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}, 0x3, 0xc05095fb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00df47980}}}, {0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}, 0xc02be17c40, 0xc05095fb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc061235740}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc061235740}, {0x40, 0xc00df47b70}, 0xc00df47ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc061235740}, {0x4e9d7a0, 0xc062d7b300}, 0xc02be17c40, 0xc041bf3938) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc061235740}, 0xc0aadfa480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc049d55c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc049d55c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811132 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06bc00, {0x63c85b8, 0xc02aa5ef30}, 0xc0a2434900, 0xc065741ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc02aa5ef30}, 0xc0a2434900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110763d35cd, 0x0, 0x0}, 0xc04f6f5010, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110763d35cd, 0x0, 0x0}, 0xc04f6f5010, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02aa5ec60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02aa5ec60}, {0xc0766b7a08, 0x660089}, 0xc0766b7a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02aa5ec60}, {0xc00102e4e0}, 0xc0a2434800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02aa5e7e0}, 0xc0a2434800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc02aa5e600}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc02aa5e600}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805657 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ecbf400, {0x63c85b8, 0xc041ec5dd0}, 0xc03c3a38f0, 0x1, 0x708017e513ce6b, 0x37e11d600, 0xc093b98960, 0xc073e36720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041ec5dd0}, 0x120, 0xc001cd7230, 0x93b98900, 0x0, 0x660069, 0xc093b98960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ecbf400, {0x63c85b8, 0xc041ec5dd0}, 0xc093b98900, {{{{0x0, 0x0}, {0xc00f1039e0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041ec5dd0}, 0x6352410, 0x4207e268) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01eb85e50, {0x63c85b8, 0xc041ec5dd0}, {0xc093b987e0, {0x174301103f6cb70c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01eb85e50, {0x63c85b8, 0xc041ec5dd0}, 0xc0519200a0, 0xc001cd79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041ec5dd0}, 0x660089, {0xc093b987e0, {0x174301103f6cb70c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ed2e800, {0x63c85b8, 0xc041ec5dd0}, 0xc0607a5900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f6cb70c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f6cb70c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041ec5d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ec5d40}, {0xc001cd93f8, 0x660089}, 0xc001cd9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041ec5d40}, {0x622b315550537955}, 0xc0607a5700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041ec5cb0}, 0xc0607a5700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}, 0xc01e670f20, 0xc03c3a3860) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}, 0x3, 0xc03de36c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc001cd9980}}}, {0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}, 0xc01e670f20, 0xc03de36c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041ec5cb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ec5cb0}, {0x40, 0xc001cd9b70}, 0xc001cd9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041ec5cb0}, {0x4e9d7a0, 0xc0607a5700}, 0xc01e670f20, 0xc03c3a3860) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041ec5cb0}, 0xc046b4a1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc093b98360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc093b98360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799377 [runnable]: sync.runtime_SemacquireMutex(0xc0620ce690, 0x8f, 0x7fda5c5b3f80) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0620ce538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0620ce560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0525f8000, {0xffffffffffffff15, 0xc0620ce6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0525f8000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0525f8000, {0xc0016bdb90, 0x32, 0x6f}, {0xc030138e00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093064898, {0xc0016bdb90, 0x32, 0x6f}, {0xc030138e00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0930631c0, {0xc030138e00, 0x74, 0x6f}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0930631c0, {0xc0016bdb90, 0x74, 0xc030138e00}, {0xc030138e00, 0xc007739a40, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093062f00, {0xc030138e00, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc072e27e28, {{0xc007739a40, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc051d75000, {{0xc0297485a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04e5e4c00, {0x63c85b8, 0xc01c9ea780}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01c9ea780}, {0x64a6560, 0xc051d75000}, {0xc0297485a0, 0x53, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01c9ea780}, {0x6466ed8, 0xc072e27b80}, {0xc0297485a0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc066a049d8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06d881000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0205cc400, {0x63c85b8, 0xc01c9ea780}, 0xc06d881000, 0xc04b110be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cc400, {0x63c85b8, 0xc01c9ea780}, 0xc06d881000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c9ea3f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c9ea3f0}, {0xc0076c88f0, 0x660089}, 0xc0076c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c9ea3f0}, {0xc00102e4e0}, 0xc06d880f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c9ea240}, 0xc06d880f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc09ed74060}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805446 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b9a2b40, {0x63c85b8, 0xc041ca13e0}, 0xc0465cbdb8, 0x1, 0x708017e142267f, 0x37e11d600, 0xc05f70b800, 0xc058e03ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041ca13e0}, 0x120, 0xc085931230, 0x5f70b7a0, 0xc0133481a0, 0x18, 0xc05f70b800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b9a2b40, {0x63c85b8, 0xc041ca13e0}, 0xc05f70b7a0, {{{{0x0, 0x0}, {0xc027444ea0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041ca13e0}, 0x6352410, 0x24ff1400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0fb950, {0x63c85b8, 0xc041ca13e0}, {0xc05f70b680, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0fb950, {0x63c85b8, 0xc041ca13e0}, 0xc046deda40, 0xc0859319f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041ca13e0}, 0x660089, {0xc05f70b680, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9bb000, {0x63c85b8, 0xc041ca13e0}, 0xc055a4bf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041ca1350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ca1350}, {0xc0859333f8, 0x660089}, 0xc085933400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041ca1350}, {0x7fe394451a98}, 0xc055a4bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041ca1290}, 0xc055a4bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}, 0xc01389a340, 0xc0465cbd88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}, 0x3, 0xc074875d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc085933980}}}, {0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}, 0xc01389a340, 0xc074875d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041ca1290}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ca1290}, {0x40, 0xc085933b70}, 0xc085933ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}, 0x20301d, 0x20301d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041ca1290}, {0x4e9d7a0, 0xc055a4bd00}, 0xc01389a340, 0xc0465cbd88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041ca1290}, 0xc02cab0b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f70afc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f70afc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802839 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021b5dc00, {0x63c85b8, 0xc05f6b53e0}, 0xc005df4600, 0xc075273f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021b5dc00, {0x63c85b8, 0xc05f6b53e0}, 0xc005df4600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110270fdf2a, 0x0, 0x0}, 0xc0554de570, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110270fdf2a, 0x0, 0x0}, 0xc0554de570, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05f6b5350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f6b5350}, {0xc076857918, 0x660089}, 0xc076857920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05f6b5350}, {0xc00102e4e0}, 0xc005df4500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05f6b52f0}, 0xc005df4500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06b3a2c60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02a290b10}, {0xc00dfd36b0, 0x24}, 0x46c7b8de8, 0xc039a76d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc0733129c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc02a290b10}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812654 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791457 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b2a90, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0539d8200, 0xc0756fc000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0539d8200, {0xc0756fc000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0539d8200, {0xc0756fc000, 0x735, 0xc0557caa00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc07b229ff0, {0xc0756fc000, 0x7eda79, 0xc0756fc005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0504d6ea0, {0xc0756fc000, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2690f30, {0xc0756fc000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc032915778, {0x631b900, 0xc0b2690f30}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc032915500, {0x7fe42ab59310, 0xc0504d6ea0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc032915500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc032915500, {0xc00dd0a000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0282d1a88, {0xc00dd0a000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc02afec088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc02afec088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc02afec660, {0x634aac0, 0xc02afec088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc02afec000, 0xc006b653c0, 0xc08734b42b, 0xc08734b42d, {0x635bec0, 0xc01ad30480}, 0xc000d6ae00, 0xc03e6f9004, 0xc08734b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc02afec000, {0x63c85b8, 0xc072701650}, 0xc006b653b0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc00406f03b, 0x8}}, 0x0, 0xc0727015f0, 0xc072701620, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc072701650}, {0x643d810, 0xc0504d6ea0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0425de5a0}, {0x643d810, 0xc0504d6ea0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0425de5a0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810363 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021a5f2c0, {0x63c85b8, 0xc076571e00}, 0xc02060f320, 0x1, 0xc027f710a8, 0x519bbc, 0xc0820f2840, 0xc072310840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc076571e00}, 0x120, 0xc027f71230, 0x820f27e0, 0x0, 0x660069, 0xc0820f2840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021a5f2c0, {0x63c85b8, 0xc076571e00}, 0xc0820f27e0, {{{{0x0, 0x0}, {0xc048130120, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc076571e00}, 0x6352410, 0x51b88ce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a30d20, {0x63c85b8, 0xc076571e00}, {0xc0820f26c0, {0x174301106cd2d628, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a30d20, {0x63c85b8, 0xc076571e00}, 0xc081f75720, 0xc027f719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc076571e00}, 0x660089, {0xc0820f26c0, {0x174301106cd2d628, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b1000, {0x63c85b8, 0xc076571e00}, 0xc088817900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cd2d628, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cd2d628, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076571d70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076571d70}, {0xc027f733f8, 0x660089}, 0xc027f73400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076571d70}, {0x0}, 0xc088817700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076571ce0}, 0xc088817700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}, 0xc042e15a80, 0xc047f8b6f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}, 0x3, 0xc07a00a5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc027f73980}}}, {0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}, 0xc042e15a80, 0xc07a00a5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076571ce0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076571ce0}, {0x40, 0xc027f73b70}, 0xc027f73ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076571ce0}, {0x4e9d7a0, 0xc088817700}, 0xc042e15a80, 0xc047f8b6f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076571ce0}, 0xc0b60aef00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01f3b86c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01f3b86c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803753 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc045785860}, 0xc02c48c6f0, 0x1, 0x0, 0x1, 0xc055ec6060, 0xc0929f2ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x360, {0x63c85b8, 0xc045785860}, 0x380, 0xc06e3c4728, 0x55ec6000, 0x0, 0x660069, 0xc055ec6060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc045785860}, 0xc055ec6000, {{{{0x0, 0x0}, {0xc04cfaa900, 0x48}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc06ce008a0, {0x63c85b8, 0xc045785860}, 0xc0021e6580, 0x2e074240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc045785860}, {0xc0460e07e0, {0x174301102f46acf5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc045785860}, 0xc0307a7900, 0xc076588ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc045785860}, 0x660089, {0xc0460e07e0, {0x174301102f46acf5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc045785860}, 0xc064951d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102f46acf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102f46acf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0457857a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0457857a0}, {0xc002fea8f0, 0x660089}, 0xc002fea8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0457857a0}, {0xc00102e4e0}, 0xc064951c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045785740}, 0xc064951c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07c6aff20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807799 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc07a433cb0}, 0xc043eb9200, 0x1, 0xc0047990a8, 0x519bbc, 0xc0774e5260, 0xc0b9b52760) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07a433cb0}, 0x120, 0xc004799230, 0x774e5200, 0x0, 0x660069, 0xc0774e5260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc07a433cb0}, 0xc0774e5200, {{{{0x0, 0x0}, {0xc06540e6c0, 0x10}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07a433cb0}, 0x63526b0, 0x4040de28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc07a433cb0}, {0xc0774e50e0, {0x174301105ac410b3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc07a433cb0}, 0xc043be8be0, 0xc0047999f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a433cb0}, 0x660089, {0xc0774e50e0, {0x174301105ac410b3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc07a433cb0}, 0xc0558e6800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105ac410b3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105ac410b3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a433c20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a433c20}, {0xc00479b3f8, 0x660089}, 0xc00479b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a433c20}, {0x0}, 0xc0558e6600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a433b90}, 0xc0558e6600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}, 0xc079bae220, 0xc0415c8a80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}, 0x3, 0xc0708b8580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00479b980}}}, {0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}, 0xc079bae220, 0xc0708b8580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a433b90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a433b90}, {0x40, 0xc00479bb70}, 0xc00479bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a433b90}, {0x4e9d7a0, 0xc0558e6600}, 0xc079bae220, 0xc0415c8a80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a433b90}, 0xc047097e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0774e4ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0774e4ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809841 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc05ca98dc0, {0x63c85b8, 0xc0b1f198c0}, 0xc0456aebe8, 0x1, 0xc01572f0a8, 0x519bbc, 0xc03cfe91a0, 0xc037e23ce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05907d200, {0x63c85b8, 0xc0b1f198c0}, 0xc05907d240, 0xc01572f1a8, 0x51, 0x70, 0xffffffffffffffff, 0xc03cfe91a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc05ca98dc0, {0x63c85b8, 0xc0b1f198c0}, 0xc03cfe90e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc040edf200, 0xd}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0b1f198c0}, 0x6352260, 0x3102f538) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc045008410, {0x63c85b8, 0xc0b1f198c0}, {0xc03cfe8fc0, {0x174301102bb82fa6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc045008410, {0x63c85b8, 0xc0b1f198c0}, 0xc079424e60, 0xc01572f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0b1f198c0}, 0x660089, {0xc03cfe8fc0, {0x174301102bb82fa6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc06ffeec00, {0x63c85b8, 0xc0b1f198c0}, 0xc07ced2c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bb82fa6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bb82fa6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0b1f197d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b1f197d0}, {0xc0157313f8, 0x660089}, 0xc015731400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0b1f197d0}, {0x7fe34af3c588}, 0xc07ced2900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0b1f196b0}, 0xc07ced2900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}, 0xc0631cffe0, 0xc0456affb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}, 0x3, 0xc099165ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc015731980}}}, {0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}, 0xc0631cffe0, 0xc099165ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0b1f196b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b1f196b0}, {0x40, 0xc015731b70}, 0xc015731ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0b1f196b0}, {0x4e9d7a0, 0xc07ced2900}, 0xc0631cffe0, 0xc0456affb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0b1f196b0}, 0xc01b1cdd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06118d8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06118d8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812653 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804037 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ffbfcc0, {0x63c85b8, 0xc044e78fc0}, 0xc0c11de8a0, 0x1, 0xc04250d0a8, 0x519bbc, 0xc0578fb5c0, 0xc055f896e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc044e78fc0}, 0x120, 0xc04250d230, 0x578fb560, 0x0, 0x660069, 0xc0578fb5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ffbfcc0, {0x63c85b8, 0xc044e78fc0}, 0xc0578fb560, {{{{0x0, 0x0}, {0xc05c2b4000, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc044e78fc0}, 0x6352410, 0x45bf52d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f553860, {0x63c85b8, 0xc044e78fc0}, {0xc0578fb440, {0x1743011032336597, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f553860, {0x63c85b8, 0xc044e78fc0}, 0xc07c811ae0, 0xc04250d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044e78fc0}, 0x660089, {0xc0578fb440, {0x1743011032336597, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030124000, {0x63c85b8, 0xc044e78fc0}, 0xc05654b700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032336597, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032336597, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044e78f30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044e78f30}, {0xc04250f3f8, 0x660089}, 0xc04250f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044e78f30}, {0x0}, 0xc05654b500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044e78cc0}, 0xc05654b500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}, 0xc0204b2c40, 0xc00ddcd308) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}, 0x3, 0xc0188c1880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04250f980}}}, {0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}, 0xc0204b2c40, 0xc0188c1880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044e78cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044e78cc0}, {0x40, 0xc04250fb70}, 0xc04250fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044e78cc0}, {0x4e9d7a0, 0xc05654b500}, 0xc0204b2c40, 0xc00ddcd308) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044e78cc0}, 0xc029acfc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0578fb320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0578fb320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801201 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc03023f230}, 0xc0621b2400, 0xc08c7d0dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc03023f230}, 0xc0621b2400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c35d1c0, 0x0, 0x0}, 0xc051714de0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c35d1c0, 0x0, 0x0}, 0xc051714de0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03023f1a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03023f1a0}, {0xc08cd73a08, 0x660089}, 0xc08cd73a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03023f1a0}, {0xc00102e4e0}, 0xc0621b2300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03023f140}, 0xc0621b2300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc03023f050}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc03023f050}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800385 [semacquire]: sync.runtime_SemacquireMutex(0xc0133489c0, 0x3, 0xc0133489c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc07e130338) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fe41ec29d40, 0x7dac, 0x7fe2c1526b80) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc020554940, 0x7fe41ec29d01, 0x7fe41ec29d60, 0x39f1, 0x39f6) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc011f082c0, {0x630dd0e, 0x39f1}, 0x0, 0xc04d42fd78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04d42fb00, {0xffffffffffffff15, 0xc07e1305c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc04d42fb00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc04d42fb00, {0xc016d11900, 0x2b, 0x76}, 0x13, 0xcd) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04d42fb00, {0xc02d2f9a00, 0x2b, 0x76}, {0xc016d11900, 0x51c9860, 0xc016d11900}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04d42fb00, {0xc02d2f9a00, 0x2b, 0x77}, {0xc016d11900, 0x2b, 0x76}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc075d99e98, {0xc02d2f9a00, 0x2b, 0x77}, {0xc016d11900, 0x2b, 0x76}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc075d987c0, {0xc016d11900, 0x76, 0x77}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc075d987c0, {0xc02d2f9a00, 0x2b, 0xc016d11900}, {0xc016d11900, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc075d98500, {0xc016d11900, 0x2b, 0x76}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0514c5608, {{0xc0ba80fb30, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc040c4f400, {{0xc0ba80fb30, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05f7b7000, {0x63c85b8, 0xc06b4b85d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06b4b85d0}, {0x64a6560, 0xc040c4f400}, {0xc0ba80fb30, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06b4b85d0}, {0x6466ed8, 0xc0514c5600}, {0xc0ba80fb30, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0654a7dd0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc064a7ff00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020abd400, {0x63c85b8, 0xc06b4b85d0}, 0xc064a7ff00, 0xc04e4cdd60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020abd400, {0x63c85b8, 0xc06b4b85d0}, 0xc064a7ff00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b4b8540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b4b8540}, {0xc0032fe8f0, 0x660089}, 0xc0032fe8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b4b8540}, {0xc00102e4e0}, 0xc064a7fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b4b84e0}, 0xc064a7fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b950}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812683 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809364 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020edc000, {0x63c85b8, 0xc04a22ab40}, 0xc055b24a00, 0xc046ded4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020edc000, {0x63c85b8, 0xc04a22ab40}, 0xc055b24a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b86b204, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b86b204, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04a22aab0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a22aab0}, {0xc01a0b13f8, 0x660089}, 0xc01a0b1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04a22aab0}, {0x7fe3488954f8}, 0xc055b24700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04a22aa20}, 0xc055b24700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}, 0xc02fa35ee0, 0xc01e9bee58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}, 0x3, 0xc0525cdd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a0b1980}}}, {0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}, 0xc02fa35ee0, 0xc0525cdd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04a22aa20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a22aa20}, {0x40, 0xc01a0b1b70}, 0xc01a0b1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04a22aa20}, {0x4e9d7a0, 0xc055b24700}, 0xc02fa35ee0, 0xc01e9bee58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04a22aa20}, 0xc04efe2240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc048eca900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc048eca900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812665 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807111 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc047e41b60}, 0xc08fd18600, 0xc03b365e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc047e41b60}, 0xc08fd18600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110572b8781, 0x0, 0x0}, 0xc08e8a54a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110572b8781, 0x0, 0x0}, 0xc08e8a54a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e41ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e41ad0}, {0xc0056093f8, 0x660089}, 0xc005609400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e41ad0}, {0x7fe38e27e4b0}, 0xc08fd18400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e41a40}, 0xc08fd18400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}, 0xc077d4c660, 0xc02620d6c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}, 0x3, 0xc0612c4f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005609980}}}, {0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}, 0xc077d4c660, 0xc0612c4f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047e41a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e41a40}, {0x40, 0xc005609b70}, 0xc005609ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047e41a40}, {0x4e9d7a0, 0xc08fd18400}, 0xc077d4c660, 0xc02620d6c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047e41a40}, 0xc04580f140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07ca20d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07ca20d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809728 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0039f7dd0}, 0xc076fdb980, 0x1, 0x708018052efec4, 0x37e11d600, 0xc0906b8720, 0xc05564d8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0039f7dd0}, 0x120, 0xc01a7fd230, 0x906b86c0, 0x0, 0x660069, 0xc0906b8720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0039f7dd0}, 0xc0906b86c0, {{{{0x0, 0x0}, {0xc059ed4fc0, 0xf3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0039f7dd0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0039f7dd0}, {0x0, {0x174301106890fdca, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0039f7dd0}, 0xc075bfd680, 0xc01a7fd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0039f7dd0}, 0x660089, {0x0, {0x174301106890fdca, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0039f7dd0}, 0xc0544c8100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106890fdca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106890fdca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301106890fdca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0039f7d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0039f7d40}, {0xc01a7ff3f8, 0x660089}, 0xc01a7ff400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0039f7d40}, {0x7fe351e6e528}, 0xc07376fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0039f7c80}, 0xc07376fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}, 0xc0649b7d40, 0xc076fdb950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}, 0x3, 0xc076c49580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a7ff980}}}, {0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}, 0xc0649b7d40, 0xc076c49580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0039f7c80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0039f7c80}, {0x40, 0xc01a7ffb70}, 0xc01a7ffab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0039f7c80}, {0x4e9d7a0, 0xc07376fe00}, 0xc0649b7d40, 0xc076fdb950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0039f7c80}, 0xc043d6dc80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0906b85a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0906b85a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800132 [runnable]: sync.runtime_SemacquireMutex(0xc079fde690, 0x8f, 0x7fd46a8c1e40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc079fde538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc079fde560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0489a3200, {0xffffffffffffff15, 0xc079fde6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0489a3200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0489a3200, {0xc0361a6a80, 0x32, 0x74}, {0xc01859ac80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d468d98, {0xc0361a6a80, 0x32, 0x74}, {0xc01859ac80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d4676c0, {0xc01859ac80, 0x74, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d4676c0, {0xc0361a6a80, 0x74, 0xc01859ac80}, {0xc01859ac80, 0xc06c9a00a0, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d467400, {0xc01859ac80, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0963ed8a8, {{0xc06c9a00a0, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05075e600, {{0xc082655680, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ff38000, {0x63c85b8, 0xc037aee2a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc037aee2a0}, {0x64a6560, 0xc05075e600}, {0xc082655680, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc037aee2a0}, {0x6466ed8, 0xc0963ed600}, {0xc082655680, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc016f08468}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc043762d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209f3400, {0x63c85b8, 0xc037aee2a0}, 0xc043762d00, 0xc0513f23c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209f3400, {0x63c85b8, 0xc037aee2a0}, 0xc043762d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc037aee210}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037aee210}, {0xc0933e48f0, 0x660089}, 0xc0933e48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc037aee210}, {0xc00102e4e0}, 0xc043762c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc037aee1b0}, 0xc043762c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d1d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812662 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219798719 [runnable]: sync.runtime_SemacquireMutex(0xc092474690, 0x8f, 0xc0924746a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0xc0924744f0, 0x11948b8) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x3b, 0x77, 0x3c, 0xc092474560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc03f08c480, {0xffffffffffffff15, 0xc0924746c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc03f08c480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc03f08c480, {0xc063ec3810, 0x32, 0x50}, {0xc023fd4600, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc075c55e98, {0xc063ec3810, 0x32, 0x50}, {0xc023fd4600, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc075c547c0, {0xc023fd4600, 0x77, 0x50}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc075c547c0, {0xc063ec3810, 0x77, 0xc023fd4600}, {0xc023fd4600, 0xc014d25780, 0x31}, 0x78) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc075c54500, {0xc023fd4600, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0769559a8, {{0xc014d25780, 0x31, 0x78}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0d7eea400, {{0xc0781d9350, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0d54c9c00, {0x63c85b8, 0xc0620a7bf0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0620a7bf0}, {0x64a6560, 0xc0d7eea400}, {0xc0781d9350, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0620a7bf0}, {0x6466ed8, 0xc076955700}, {0xc0781d9350, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc03bfb3950}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05955d300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020579800, {0x63c85b8, 0xc0620a7bf0}, 0xc05955d300, 0xc0468854a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020579800, {0x63c85b8, 0xc0620a7bf0}, 0xc05955d300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0620a7b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0620a7b60}, {0xc0110d68f0, 0x660089}, 0xc0110d68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0620a7b60}, {0xc00102e4e0}, 0xc05955ce00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0620a7ad0}, 0xc05955ce00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03c4c6cc0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807552 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030065c00, {0x63c85b8, 0xc02d17c7e0}, 0xc09a1cbc00, 0xc04328aa00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc02d17c7e0}, 0xc09a1cbc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058e778ad, 0x0, 0x0}, 0xc092101eb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058e778ad, 0x0, 0x0}, 0xc092101eb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02d17c750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d17c750}, {0xc07ce43a08, 0x660089}, 0xc07ce43a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02d17c750}, {0xc00102e4e0}, 0xc09a1cbb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02d17c6f0}, 0xc09a1cbb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc02d17c660}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc02d17c660}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811049 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc026bed3b0}, 0xc0cae76de0, 0x1, 0xc01a5930a8, 0x519bbc, 0xc000f69b60, 0xc052a9cba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc026bed3b0}, 0x120, 0xc01a593230, 0xf69b00, 0x0, 0x660069, 0xc000f69b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc026bed3b0}, 0xc000f69b00, {{{{0x0, 0x0}, {0xc01d15e120, 0xc}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc026bed3b0}, 0x6352410, 0x3c750398) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc026bed3b0}, {0xc000f698c0, {0x174301106bf9e35a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc026bed3b0}, 0xc07236bc20, 0xc01a5939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc026bed3b0}, 0x660089, {0xc000f698c0, {0x174301106bf9e35a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc026bed3b0}, 0xc04c0ad800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bf9e35a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bf9e35a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc026bed230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc026bed230}, {0xc01a5953f8, 0x660089}, 0xc01a595400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc026bed230}, {0x0}, 0xc04c0ad600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc026bed0e0}, 0xc04c0ad600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}, 0xc028541880, 0xc06ddab9e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}, 0x3, 0xc027e8d4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a595980}}}, {0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}, 0xc028541880, 0xc027e8d4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc026bed0e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc026bed0e0}, {0x40, 0xc01a595b70}, 0xc01a595ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc026bed0e0}, {0x4e9d7a0, 0xc04c0ad600}, 0xc028541880, 0xc06ddab9e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc026bed0e0}, 0xc037f36b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc000f69680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc000f69680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801601 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0310bf000, {0x63c85b8, 0xc072be0ae0}, 0xc04bb8e500, 0xc06477de00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310bf000, {0x63c85b8, 0xc072be0ae0}, 0xc04bb8e500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff88a39c4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff88a39c4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072be0a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072be0a20}, {0xc04dcf33f8, 0x660089}, 0xc04dcf3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072be0a20}, {0x7fe3f3252190}, 0xc04bb8e300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072be0960}, 0xc04bb8e300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}, 0xc0b844bfa0, 0xc011c019f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}, 0x3, 0xc00db82300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04dcf3980}}}, {0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}, 0xc0b844bfa0, 0xc00db82300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc072be0960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072be0960}, {0x40, 0xc04dcf3b70}, 0xc04dcf3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc072be0960}, {0x4e9d7a0, 0xc04bb8e300}, 0xc0b844bfa0, 0xc011c019f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc072be0960}, 0xc023841ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0435b8000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0435b8000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806537 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024c49900, {0x63c85b8, 0xc08c118690}, 0xc08f564858, 0x1, 0xc00e1170a8, 0x519bbc, 0xc01df876e0, 0xc046cde060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08c118690}, 0x120, 0xc00e117230, 0x1df87680, 0x0, 0x660069, 0xc01df876e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024c49900, {0x63c85b8, 0xc08c118690}, 0xc01df87680, {{{{0x0, 0x0}, {0xc047aeab40, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08c118690}, 0x6352410, 0x5449a748) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024c053b0, {0x63c85b8, 0xc08c118690}, {0xc01df87440, {0x174301104e81a176, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024c053b0, {0x63c85b8, 0xc08c118690}, 0xc03b27e780, 0xc00e1179f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08c118690}, 0x660089, {0xc01df87440, {0x174301104e81a176, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024ca9800, {0x63c85b8, 0xc08c118690}, 0xc058b69500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e81a176, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e81a176, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08c118600}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08c118600}, {0xc00e1193f8, 0x660089}, 0xc00e119400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08c118600}, {0x0}, 0xc058b69300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08c118570}, 0xc058b69300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}, 0xc083799ec0, 0xc017136960) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}, 0x3, 0xc087191280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e119980}}}, {0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}, 0xc083799ec0, 0xc087191280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08c118570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08c118570}, {0x40, 0xc00e119b70}, 0xc00e119ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08c118570}, {0x4e9d7a0, 0xc058b69300}, 0xc083799ec0, 0xc017136960) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08c118570}, 0xc0415fa360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01df87320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01df87320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809932 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc031194500, {0x63c85b8, 0xc022f8d5c0}, 0xc0615a23d8, 0x1, 0xc019e7b0a8, 0x519bbc, 0xc0761a3e00, 0xc04ceb1ce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc036cb7680, {0x63c85b8, 0xc022f8d5c0}, 0xc0761a3e00, 0xc019e7b1a8, 0x33, 0x40, 0x2, 0xc0761a3e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc031194500, {0x63c85b8, 0xc022f8d5c0}, 0xc0761a3d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc079b02fc0, 0x7}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc022f8d5c0}, 0x6352260, 0x450cbe58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc03107f9f0, {0x63c85b8, 0xc022f8d5c0}, {0xc0761a3c20, {0x174301102d39292f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc03107f9f0, {0x63c85b8, 0xc022f8d5c0}, 0xc051998280, 0xc019e7b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022f8d5c0}, 0x660089, {0xc0761a3c20, {0x174301102d39292f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031189400, {0x63c85b8, 0xc022f8d5c0}, 0xc0824d3800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d39292f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d39292f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022f8d410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022f8d410}, {0xc019e7d3f8, 0x660089}, 0xc019e7d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022f8d410}, {0xc0351a72c0}, 0xc0824d3600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022f8cf90}, 0xc0824d3600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}, 0xc085874380, 0xc03d4ae030) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}, 0x3, 0xc03937db80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc019e7d980}}}, {0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}, 0xc085874380, 0xc03937db80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022f8cf90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022f8cf90}, {0x40, 0xc019e7db70}, 0xc019e7dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022f8cf90}, {0x4e9d7a0, 0xc0824d3600}, 0xc085874380, 0xc03d4ae030) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022f8cf90}, 0xc039d510e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0761a3b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0761a3b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807565 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc0160c8420}, 0xc020230720, 0x1, 0xc005d990a8, 0x519bbc, 0xc064b79aa0, 0xc092eeca00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0734ee780, {0x63c85b8, 0xc0160c8420}, 0xc073da67e0, 0xc005d991a8, 0x32, 0x40, 0x0, 0xc064b79aa0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc0160c8420}, 0xc064b79a40, {{{{0xc0a5596120, 0x17}, {0xc07d95cb40, 0x30}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x40, {0x63c85b8, 0xc0160c8420}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc0160c8420}, {0xc08ececc60, {0x174301100997f8c8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc0160c8420}, 0xc04703dcc0, 0xc005d999f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0160c8420}, 0x660089, {0xc08ececc60, {0x174301100997f8c8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0160c8420}, 0xc09849ae00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100997f8c8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100997f8c8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0160c8360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0160c8360}, {0xc005d9b3f8, 0x660089}, 0xc005d9b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0160c8360}, {0x0}, 0xc09849ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0160c81b0}, 0xc09849ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}, 0xc01cd30f80, 0xc03ac6ef78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}, 0x3, 0xc0635d6980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005d9b980}}}, {0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}, 0xc01cd30f80, 0xc0635d6980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0160c81b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0160c81b0}, {0x40, 0xc005d9bb70}, 0xc005d9bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0160c81b0}, {0x4e9d7a0, 0xc09849ac00}, 0xc01cd30f80, 0xc03ac6ef78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0160c81b0}, 0xc04bf1e5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08ececa20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08ececa20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803306 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc07503c750}, 0xc0573a8d98, 0x1, 0x0, 0x1, 0xc0328e4cc0, 0xc07d48da40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07503c750}, 0x120, 0xc074f92728, 0x328e4c60, 0x0, 0x660069, 0xc0328e4cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc07503c750}, 0xc0328e4c60, {{{{0x0, 0x0}, {0xc0473226c0, 0xb}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07503c750}, 0x6352410, 0x3ad19fd8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc07503c750}, {0xc0328e4a20, {0x1743011019292be5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc07503c750}, 0xc06b233ea0, 0xc08da96ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07503c750}, 0x660089, {0xc0328e4a20, {0x1743011019292be5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc07503c750}, 0xc0c5996100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07503c6c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07503c6c0}, {0xc07d2568f0, 0x660089}, 0xc07d2568f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07503c6c0}, {0x24}, 0xc0c5996000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07503c660}, 0xc0c5996000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07503c450}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807354 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fda5000, {0x63c85b8, 0xc07c076240}, 0xc090f65900, 0xc05f536c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fda5000, {0x63c85b8, 0xc07c076240}, 0xc090f65900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110571b9d18, 0x0, 0x0}, 0xc08cc9fe40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110571b9d18, 0x0, 0x0}, 0xc08cc9fe40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c0761b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c0761b0}, {0xc00561d3f8, 0x660089}, 0xc00561d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c0761b0}, {0xc04ed7c140}, 0xc090f65700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c076120}, 0xc090f65700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}, 0xc076f0c460, 0xc050c59ce0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}, 0x3, 0xc0a3efde80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00561d980}}}, {0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}, 0xc076f0c460, 0xc0a3efde80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c076120}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c076120}, {0x40, 0xc00561db70}, 0xc00561dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c076120}, {0x4e9d7a0, 0xc090f65700}, 0xc076f0c460, 0xc050c59ce0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c076120}, 0xc0171c0600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc084aaad80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc084aaad80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809017 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc09bf29f50}, 0xc0584b8d38, 0x1, 0xc01a10b0a8, 0x519bbc, 0xc017109e60, 0xc024004420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc09bf29f50}, 0x1e0, 0xc01a10b230, 0x17109e00, 0x0, 0x660069, 0xc017109e60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc09bf29f50}, 0xc017109e00, {{{{0x0, 0x0}, {0xc01a600240, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc09bf29f50}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc09bf29f50}, {0xc081f6f200, {0x17430110617803ba, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc09bf29f50}, 0xc079424b40, 0xc01a10b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc09bf29f50}, 0x660089, {0xc081f6f200, {0x17430110617803ba, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc09bf29f50}, 0xc051eb0700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110617803ba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110617803ba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09bf29ec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09bf29ec0}, {0xc01a10d3f8, 0x660089}, 0xc01a10d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09bf29ec0}, {0x7fe3f1f7fc30}, 0xc051eb0500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09bf29e30}, 0xc051eb0500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}, 0xc0ce3149e0, 0xc0681a3200) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}, 0x3, 0xc043035840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a10d980}}}, {0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}, 0xc0ce3149e0, 0xc043035840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09bf29e30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09bf29e30}, {0x40, 0xc01a10db70}, 0xc01a10dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09bf29e30}, {0x4e9d7a0, 0xc051eb0500}, 0xc0ce3149e0, 0xc0681a3200) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09bf29e30}, 0xc04de16b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0890cbe60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0890cbe60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803389 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc0734aaab0}, 0xc0276eda40, 0x1, 0xc077a450a8, 0x519bbc, 0xc0449dc600, 0xc04afa8600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0734aaab0}, 0x1e0, 0xc077a45230, 0x449dc5a0, 0x0, 0x660069, 0xc0449dc600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc0734aaab0}, 0xc0449dc5a0, {{{{0x0, 0x0}, {0xc0748c4d80, 0x53}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0734aaab0}, 0x1, 0x77a458b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc0734aaab0}, {0xc03fb6eb40, {0x174301102edfca1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc0734aaab0}, 0xc0499d70e0, 0xc077a459f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0734aaab0}, 0x660089, {0xc03fb6eb40, {0x174301102edfca1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0734aaab0}, 0xc0b003fc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102edfca1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102edfca1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0734aaa20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0734aaa20}, {0xc077a473f8, 0x660089}, 0xc077a47400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0734aaa20}, {0x4d939c}, 0xc0b003f900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0734aa990}, 0xc0b003f900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}, 0xc01dda3d80, 0xc03c696a08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}, 0x3, 0xc047557380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc077a47980}}}, {0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}, 0xc01dda3d80, 0xc047557380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0734aa990}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0734aa990}, {0x40, 0xc077a47b70}, 0xc077a47ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0734aa990}, {0x4e9d7a0, 0xc0b003f900}, 0xc01dda3d80, 0xc03c696a08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0734aa990}, 0xc027437c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03fb6ea20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03fb6ea20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803878 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031189400, {0x63c85b8, 0xc06dcbaea0}, 0xc0651b5900, 0xc039b4a780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031189400, {0x63c85b8, 0xc06dcbaea0}, 0xc0651b5900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d39292f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d39292f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06dcbae10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dcbae10}, {0xc06781d3f8, 0x660089}, 0xc06781d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06dcbae10}, {0x7fe3b8ab22d0}, 0xc0651b5700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06dcbad50}, 0xc0651b5700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}, 0xc01a508f60, 0xc04b9a65b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}, 0x3, 0xc027c35a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06781d980}}}, {0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}, 0xc01a508f60, 0xc027c35a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06dcbad50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dcbad50}, {0x40, 0xc06781db70}, 0xc06781dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06dcbad50}, {0x4e9d7a0, 0xc0651b5700}, 0xc01a508f60, 0xc04b9a65b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06dcbad50}, 0xc02d961020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04ceb19e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04ceb19e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219782318 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff360c0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc065ae1800, 0xc05637d800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc065ae1800, {0xc05637d800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc065ae1800, {0xc05637d800, 0x735, 0xc056349540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01d95b970, {0xc05637d800, 0x7eda79, 0xc05637d805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc05b00d5c0, {0xc05637d800, 0x4a7cec0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0731135c0, {0xc05637d800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc058a16978, {0x631b900, 0xc0731135c0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc058a16700, {0x7fe42ab59310, 0xc05b00d5c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc058a16700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc058a16700, {0xc03df30000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc037363f38, {0xc03df30000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc09b0ca088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc09b0ca088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc09b0ca660, {0x634aac0, 0xc09b0ca088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc09b0ca000, 0xc08fb83540, 0xc0949c142b, 0xc0949c142d, {0x635bec0, 0xc03b36a090}, 0xc000d6ae00, 0xc0302938c8, 0xc0949c1429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc09b0ca000, {0x63c85b8, 0xc0c1cf62a0}, 0xc08fb83530, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc00f520038, 0x8}}, 0x0, 0xc0c1cf6240, 0xc0c1cf6270, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0c1cf62a0}, {0x643d810, 0xc05b00d5c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc047aed530}, {0x643d810, 0xc05b00d5c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc047aed530}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811306 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc05f8ff350}, 0xc02b1cd368, 0x1, 0xc0204c70a8, 0x519bbc, 0xc08c3b36e0, 0xc0d43ea7e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc05f8ff350}, 0x1e0, 0xc0204c7230, 0x8c3b3680, 0x0, 0x660069, 0xc08c3b36e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc05f8ff350}, 0xc08c3b3680, {{{{0x0, 0x0}, {0xc0504e4900, 0x114}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc05f8ff350}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc05f8ff350}, {0xc05073c6c0, {0x174301107502f378, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc05f8ff350}, 0xc0129af180, 0xc0204c79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05f8ff350}, 0x660089, {0xc05073c6c0, {0x174301107502f378, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc05f8ff350}, 0xc0862ad900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107502f378, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107502f378, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05f8ff2c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f8ff2c0}, {0xc0204c93f8, 0x660089}, 0xc0204c9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05f8ff2c0}, {0xc0015221e0}, 0xc0862ad700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05f8ff230}, 0xc0862ad700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}, 0xc00cd97080, 0xc03fdf54e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}, 0x3, 0xc042b0bb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0204c9980}}}, {0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}, 0xc00cd97080, 0xc042b0bb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05f8ff230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f8ff230}, {0x40, 0xc0204c9b70}, 0xc0204c9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05f8ff230}, {0x4e9d7a0, 0xc0862ad700}, 0xc00cd97080, 0xc03fdf54e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05f8ff230}, 0xc0396c8de0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05073c5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05073c5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219789714 [runnable]: sync.runtime_SemacquireMutex(0xc08458ce20, 0x6c, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xf, 0xc0574aa000) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01e9c1080, {0x804f921, 0x466a}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0574aa000, {0xffffffffffffff15, 0xc08458d050}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0574aa000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0574aa000, {0xc0147d8af0, 0x4b, 0x4b}, {0xc04cc1ee00, 0x4b, 0x6e}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d267298, {0xc0147d8af0, 0x4b, 0x4b}, {0xc04cc1ee00, 0x4b, 0x6e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d265bc0, {0xc04cc1ee00, 0x4a2372, 0x4b}, 0x464ed60, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d265bc0, {0xc0147d8af0, 0x4b, 0x6e}, {0xc04cc1ee00, 0x4a, 0xa0}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d265900, {0xc04cc1ee00, 0x4b, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc044841188, {{0xc06add0320, 0x4a, 0xa0}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc049d86700, {{0xc06add0320, 0x4a, 0xa0}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc081b47400, {0x63c85b8, 0xc06c1f5da0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06c1f5da0}, {0x64a6560, 0xc049d86700}, {0xc06add0320, 0xc08458dda0, 0xa0}, {0x174301109e48cd28, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06c1f5da0}, {0x6466ed8, 0xc044841180}, {0xc06add0320, 0x4a, 0xa0}, {0x174301109e48cd28, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c5ecf48}, {{0x174301109e48cd28, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109e48cd28, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc07314e200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bc163228, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc000e4e800, {0x63c85b8, 0xc06c1f5da0}, 0xc07314e200, 0xc043693220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000e4e800, {0x63c85b8, 0xc06c1f5da0}, 0xc07314e200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109e48cd28, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109e48cd28, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06c1f5ce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c1f5ce0}, {0xc084593270, 0x660089}, 0xc084593278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06c1f5ce0}, {0x24}, 0xc07314e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06c1f5c80}, 0xc07314e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc09f7d1590, {0x63c85b8, 0xc06c1f5b90}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc09f7d1590, {0x63c85b8, 0xc06c1f5b90}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc0cc38af50, {0x63c85b8, 0xc06c1f5b90}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc08300c000, {0x63c85b8, 0xc06c1f5b90}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc030bfd7c0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc03d84ae40) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc04783eae0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).next(0xc062e70160) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:374 +0x7d github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc062e70160) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:189 +0x32c github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc03d84ae80) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0bf82ef00) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc03977a0b0, {0x63c8510, 0xc03d84ae00}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc05c74a5a0, {0x63c8510, 0xc03d84ae00}, 0xc0420d3600) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc03d84ac00}, 0xc00f401ea0, 0xc0420d3600, 0xc063154680, 0xc08e15b500, 0xc040b2bce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc03d84ac00}, 0xc040b2b9f0, 0xc00f401ea0, 0xc068925140, {{0x63ca3e8, 0xc063154580}, 0x0}, 0xc08e15b500) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc040b2b300, {0x63c8510, 0xc03d84ac00}, 0xc040b2b9f0, 0xc03d84ac00, {0x7fe42b790cf0, 0xc068925140}, 0x0, 0xc04783e958) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc040b2b300, {0x63c8510, 0xc03d84ac00}, 0xc040b2b9f0, {0x7fe42b790cf0, 0xc068925140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc040b2b300, {0x63c85b8, 0xc06c1f4f90}, {{0x63f8d78, 0xc071ca09b0}, {0xc06cbf96b8, 0x8d}, 0x2, 0x1}, 0xc0556e7680, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc040b2b300, {0x63c85b8, 0xc06c1f4f90}, {{0x63f8d78, 0xc071ca09b0}, {0xc06cbf96b8, 0x8d}, 0x2, 0x1}, 0xc0556e7680, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc040b2b300, {0x63c85b8, 0xc06c1f4f90}, {0xc0556e7680, {0xc022e0d540, 0x2, 0x2}, {0xc0748565c4, 0x6, 0x6}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0b03200e4, 0x0}, 0x0, {0xbd68a06, 0xedb7a8351, 0x0}, 0x1}, 0xc040b2b300, 0xc084599980, 0xc084599970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc040b2b300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc040b2b300, {0x63c8510, 0xc075c96600}, 0xc0588fbdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04e7175e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc075c96600}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04e7175e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812763 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219141871 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa91f0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc070d60000, 0xc0cad7ca80, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc070d60000, {0xc0cad7ca80, 0x91d, 0x91d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc070d60000, {0xc0cad7ca80, 0xc0cad7ca85, 0x62}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc05f949ea0, {0xc0cad7ca80, 0x1d7b1c5, 0xc0796be788}) GOROOT/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc02d07f6f8, {0xc0cad7ca80, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc007a73778, {0x631b900, 0xc02d07f6f8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc007a73500, {0x63290e0, 0xc05f949ea0}, 0x2) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc007a73500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc007a73500, {0xc06ede0000, 0x8000, 0x1}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc040d54de0, {0xc0da3859a0, 0x9, 0xc055fc5b00}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc040d54de0}, {0xc0da3859a0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0da3859a0, 0x9, 0xc055fc59b0}, {0x631b600, 0xc040d54de0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0da385960) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc06da0be00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1498 +0x41f created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:365 +0x194f goroutine 219726544 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc08a199550, 0x22) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc029c0c000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc029c0c660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0571a9300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0571a9300, {0x63c8510, 0xc08a199580}, 0xc06b41ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0826b1120) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc08a199580}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0826b1120) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219805343 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a5e3040, {0x63c85b8, 0xc06f159320}, 0xc01ec01e60, 0x1, 0xc078bd90a8, 0x519bbc, 0xc057f018c0, 0xc0516978c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06f159320}, 0x1e0, 0xc078bd9230, 0x57f01860, 0xc0116949c0, 0x18, 0xc057f018c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a5e3040, {0x63c85b8, 0xc06f159320}, 0xc057f01860, {{{{0x0, 0x0}, {0xc058f8fd40, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06f159320}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a5d69b0, {0x63c85b8, 0xc06f159320}, {0xc05bbd4d80, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a5d69b0, {0x63c85b8, 0xc06f159320}, 0xc04b386140, 0xc078bd99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f159320}, 0x660089, {0xc05bbd4d80, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a61a400, {0x63c85b8, 0xc06f159320}, 0xc059e96b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f159290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f159290}, {0xc078bdb3f8, 0x660089}, 0xc078bdb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f159290}, {0x70}, 0xc059e96900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f159200}, 0xc059e96900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}, 0xc00af91a00, 0xc032d859b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}, 0x3, 0xc06c1a4900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078bdb980}}}, {0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}, 0xc00af91a00, 0xc06c1a4900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f159200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f159200}, {0x40, 0xc078bdbb70}, 0xc078bdbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f159200}, {0x4e9d7a0, 0xc059e96900}, 0xc00af91a00, 0xc032d859b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f159200}, 0xc04967f8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b776120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b776120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812298 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03005e780, {0x63c85b8, 0xc079b24e10}, 0xc024450c00, 0x1, 0xc0387790a8, 0x519bbc, 0xc049b48f60, 0xc0709049c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc079b24e10}, 0x1e0, 0xc038779230, 0x49b48f00, 0x0, 0x660069, 0xc049b48f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03005e780, {0x63c85b8, 0xc079b24e10}, 0xc049b48f00, {{{{0x0, 0x0}, {0xc06b0c5e60, 0x10}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc079b24e10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030007040, {0x63c85b8, 0xc079b24e10}, {0xc09019f200, {0x174301107b6e3d85, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030007040, {0x63c85b8, 0xc079b24e10}, 0xc0123eba40, 0xc0387799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc079b24e10}, 0x660089, {0xc09019f200, {0x174301107b6e3d85, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc079b24e10}, 0xc05f880800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b6e3d85, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b6e3d85, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079b24d80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079b24d80}, {0xc03877b3f8, 0x660089}, 0xc03877b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079b24d80}, {0x0}, 0xc05f880600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079b24cf0}, 0xc05f880600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}, 0xc078fbd4c0, 0xc024450eb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}, 0x3, 0xc05c936a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03877b980}}}, {0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}, 0xc078fbd4c0, 0xc05c936a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079b24cf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079b24cf0}, {0x40, 0xc03877bb70}, 0xc03877bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079b24cf0}, {0x4e9d7a0, 0xc05f880600}, 0xc078fbd4c0, 0xc024450eb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079b24cf0}, 0xc057559c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc09019f0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc09019f0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812517 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807378 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc02f040300}, 0xc058daf050, 0x1, 0xc00368f0a8, 0x519bbc, 0xc0b9b52240, 0xc03c14ec00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05918aea0, {0x63c85b8, 0xc02f040300}, 0xc05918af10, 0xc00368f1a8, 0x41, 0x60, 0xffffffffffffffff, 0xc0b9b52240, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc02f040300}, 0xc0b9b52000, {{{{0x0, 0x0}, {0xc038848360, 0x9}}, {{0xc017662ea0, 0xd}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02f040300}, 0x63521a0, 0x1b928fc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc02f040300}, {0xc07c141680, {0x1743010fedf30874, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc02f040300}, 0xc02bc7eaa0, 0xc00368f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02f040300}, 0x660089, {0xc07c141680, {0x1743010fedf30874, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc02f040300}, 0xc08d515300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fedf30874, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fedf30874, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f040120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f040120}, {0xc0036913f8, 0x660089}, 0xc003691400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f040120}, {0x3feffffaab5c24cd}, 0xc08d515100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039393f20}, 0xc08d515100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}, 0xc06d002560, 0xc02d7e3278) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}, 0x3, 0xc0594ae5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003691980}}}, {0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}, 0xc06d002560, 0xc0594ae5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc039393f20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039393f20}, {0x40, 0xc003691b70}, 0xc003691ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc039393f20}, {0x4e9d7a0, 0xc08d515100}, 0xc06d002560, 0xc02d7e3278) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc039393f20}, 0xc051a1d0e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07c141560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07c141560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812591 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801194 [runnable]: sync.runtime_SemacquireMutex(0xc0521e9198, 0x8f, 0x7fd58523e740) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0521e9040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0521e9068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc03b8c1b00, {0xffffffffffffff15, 0xc0521e91c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc03b8c1b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc03b8c1b00, {0xc044683e80, 0x32, 0x74}, {0xc03d9e7080, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc08b7a5298, {0xc044683e80, 0x32, 0x74}, {0xc03d9e7080, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc08b7a3bc0, {0xc03d9e7080, 0x72, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc08b7a3bc0, {0xc044683e80, 0x72, 0xc03d9e7080}, {0xc03d9e7080, 0xc0400cb3e0, 0x31}, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc08b7a3900, {0xc03d9e7080, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc054a76828, {{0xc0400cb3e0, 0x31, 0x60}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc045600700, {{0xc0dde2d410, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04faff400, {0x63c85b8, 0xc03f8654a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03f8654a0}, {0x64a6560, 0xc045600700}, {0xc0dde2d410, 0x0, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03f8654a0}, {0x6466ed8, 0xc054a76580}, {0xc0dde2d410, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc071e6f458}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03399a500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c25400, {0x63c85b8, 0xc03f8654a0}, 0xc03399a500, 0xc07ad90780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c25400, {0x63c85b8, 0xc03f8654a0}, 0xc03399a500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03f865410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f865410}, {0xc0521ef3f8, 0x660089}, 0xc0521ef400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03f865410}, {0x8ec1cbf0080a2a88}, 0xc05b5d7e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03f865380}, 0xc05b5d7e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}, 0xc0210d9ea0, 0xc03b543338) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}, 0x3, 0xc01da192c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0521ef980}}}, {0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}, 0xc0210d9ea0, 0xc01da192c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03f865380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f865380}, {0x40, 0xc0521efb70}, 0xc0521efab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03f865380}, {0x4e9d7a0, 0xc05b5d7e00}, 0xc0210d9ea0, 0xc03b543338) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03f865380}, 0xc037ca9380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc067e0b7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc067e0b7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804192 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6000, {0x63c85b8, 0xc05ca7c360}, 0xc0444da570, 0x1, 0xc07540d0a8, 0x519bbc, 0xc060fdd800, 0xc04dd67140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05ca7c360}, 0x120, 0xc07540d230, 0x60fdd7a0, 0x0, 0x660069, 0xc060fdd800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6000, {0x63c85b8, 0xc05ca7c360}, 0xc060fdd7a0, {{{{0x0, 0x0}, {0xc03b87d200, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05ca7c360}, 0x6352410, 0x45bf59e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddb30, {0x63c85b8, 0xc05ca7c360}, {0xc060fdd680, {0x17430110306734ae, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc05ca7c360}, 0xc06f966320, 0xc07540d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05ca7c360}, 0x660089, {0xc060fdd680, {0x17430110306734ae, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc05ca7c360}, 0xc0839e6600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05ca7c2d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05ca7c2d0}, {0xc07540f3f8, 0x660089}, 0xc07540f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05ca7c2d0}, {0x7fe1f32ab338}, 0xc0839e6300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05ca7c240}, 0xc0839e6300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}, 0xc08cfa7e40, 0xc02a063a88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}, 0x3, 0xc072696780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07540f980}}}, {0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}, 0xc08cfa7e40, 0xc072696780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05ca7c240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05ca7c240}, {0x40, 0xc07540fb70}, 0xc07540fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05ca7c240}, {0x4e9d7a0, 0xc0839e6300}, 0xc08cfa7e40, 0xc02a063a88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05ca7c240}, 0xc077062900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06101a480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06101a480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801491 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc073d66ea0}, 0xc04ce86e00, 0xc0568e25a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc073d66ea0}, 0xc04ce86e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c6be350, 0x0, 0x0}, 0xc04d398210, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c6be350, 0x0, 0x0}, 0xc04d398210, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073d66e10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d66e10}, {0xc0421613f8, 0x660089}, 0xc042161400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073d66e10}, {0xc0628311e8}, 0xc04ce86c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073d66d80}, 0xc04ce86c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}, 0xc019d660e0, 0xc066f29a58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}, 0x3, 0xc06df03a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042161980}}}, {0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}, 0xc019d660e0, 0xc06df03a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073d66d80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d66d80}, {0x40, 0xc042161b70}, 0xc042161ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073d66d80}, {0x4e9d7a0, 0xc04ce86c00}, 0xc019d660e0, 0xc066f29a58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073d66d80}, 0xc039b91200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05b6218c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05b6218c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811977 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc035375f20}, 0xc04fe47fb0, 0x1, 0x0, 0x1, 0xc0be725a40, 0xc03019a840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc035375f20}, 0x120, 0xc03516c728, 0xbe7259e0, 0x0, 0x660069, 0xc0be725a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc035375f20}, 0xc0be7259e0, {{{{0x0, 0x0}, {0xc0b70a2b40, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc035375f20}, 0x6352410, 0x204d2330) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc035375f20}, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc035375f20}, 0xc02d9cd180, 0xc07d93eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc035375f20}, 0x660089, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc035375f20}, 0xc0827a4f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc035375e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc035375e90}, {0xc0032fa8f0, 0x660089}, 0xc0032fa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc035375e90}, {0xc00102e4e0}, 0xc0827a4e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc035375e30}, 0xc0827a4e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01f1b6a80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811618 [runnable]: sync.runtime_SemacquireMutex(0xc034e97198, 0x8f, 0x7fe01aa62880) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc034e97040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x21, 0x18, 0x21, 0xc034e97068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc052279b00, {0xffffffffffffff15, 0xc034e971c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc052279b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc052279b00, {0xc062ea92c0, 0x1f, 0x52}, {0xc0157f4680, 0x1f, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092d87298, {0xc062ea92c0, 0x1f, 0x52}, {0xc0157f4680, 0x1f, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092d85bc0, {0xc0157f4680, 0x74, 0x52}, 0x1f, 0x1f) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092d85bc0, {0xc062ea92c0, 0x74, 0xc0157f4680}, {0xc0157f4680, 0xc0767e2a90, 0x1e}, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092d85900, {0xc0157f4680, 0x1f, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0601e39a8, {{0xc0767e2a90, 0x1e, 0xd0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0664b9c00, {{0xc04835ac30, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ff38400, {0x63c85b8, 0xc0459cb470}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0459cb470}, {0x64a6560, 0xc0664b9c00}, {0xc04835ac30, 0x3d3d67432f576a6e, 0x18}, {0x1743011077a34578, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0459cb470}, {0x6466ed8, 0xc0601e3700}, {0xc04835ac30, 0x17, 0x18}, {0x1743011077a34578, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc05f7e2270}, {{0x1743011077a34578, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011077a34578, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc042e0b500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174301109570aa78, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02fd42c00, {0x63c85b8, 0xc0459cb470}, 0xc042e0b500, 0xc05f864820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fd42c00, {0x63c85b8, 0xc0459cb470}, 0xc042e0b500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077a34578, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077a34578, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0459cb3e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0459cb3e0}, {0xc034e9d3f8, 0x660089}, 0xc034e9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0459cb3e0}, {0xa00a0000100800}, 0xc042e0b200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0459cb350}, 0xc042e0b200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}, 0xc0324b2860, 0xc03045dba8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}, 0x3, 0xc025f91d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034e9d980}}}, {0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}, 0xc0324b2860, 0xc025f91d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0459cb350}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0459cb350}, {0x40, 0xc034e9db70}, 0xc034e9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0459cb350}, {0x4e9d7a0, 0xc042e0b200}, 0xc0324b2860, 0xc03045dba8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0459cb350}, 0xc04316cb40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04e3bc240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04e3bc240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803377 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc03ccb6390}, 0xc0286ff3e0, 0x1, 0xc0762c90a8, 0x519bbc, 0xc044dcf8c0, 0xc046f7a420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03ccb6390}, 0x1e0, 0xc0762c9230, 0x44dcf860, 0x0, 0x660069, 0xc044dcf8c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc03ccb6390}, 0xc044dcf860, {{{{0x0, 0x0}, {0xc075bc0480, 0x3e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03ccb6390}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc03ccb6390}, {0xc036fcf320, {0x174301102bed681e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc03ccb6390}, 0xc06477cf00, 0xc0762c99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03ccb6390}, 0x660089, {0xc036fcf320, {0x174301102bed681e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03ccb6390}, 0xc04f236300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bed681e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bed681e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03ccb6300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ccb6300}, {0xc0762cb3f8, 0x660089}, 0xc0762cb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03ccb6300}, {0xc06f830f40}, 0xc04f236100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03ccb6270}, 0xc04f236100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}, 0xc0433d25a0, 0xc037694588) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}, 0x3, 0xc0549b06c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0762cb980}}}, {0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}, 0xc0433d25a0, 0xc0549b06c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03ccb6270}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ccb6270}, {0x40, 0xc0762cbb70}, 0xc0762cbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03ccb6270}, {0x4e9d7a0, 0xc04f236100}, 0xc0433d25a0, 0xc037694588) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03ccb6270}, 0xc01bbc6a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc036fceea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc036fceea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812697 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812695 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805728 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc04c5e2d80}, 0xc04bf53620, 0x1, 0xc0073070a8, 0x519bbc, 0xc084c6d4a0, 0xc04afc8de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04c5e2d80}, 0x120, 0xc007307230, 0x84c6d440, 0x0, 0x660069, 0xc084c6d4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc04c5e2d80}, 0xc084c6d440, {{{{0x0, 0x0}, {0xc04bfded80, 0x24}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04c5e2d80}, 0x6352410, 0x4207ec48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc04c5e2d80}, {0xc084c6d320, {0x174301104bee1d80, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc04c5e2d80}, 0xc0858240a0, 0xc0073079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c5e2d80}, 0x660089, {0xc084c6d320, {0x174301104bee1d80, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc04c5e2d80}, 0xc09cdb5500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104bee1d80, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104bee1d80, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c5e2cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c5e2cf0}, {0xc0073093f8, 0x660089}, 0xc007309400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c5e2cf0}, {0x0}, 0xc09cdb5300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c5e2c60}, 0xc09cdb5300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}, 0xc0762ae6e0, 0xc08aa69a10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}, 0x3, 0xc070fe31c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007309980}}}, {0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}, 0xc0762ae6e0, 0xc070fe31c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04c5e2c60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c5e2c60}, {0x40, 0xc007309b70}, 0xc007309ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04c5e2c60}, {0x4e9d7a0, 0xc09cdb5300}, 0xc0762ae6e0, 0xc08aa69a10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04c5e2c60}, 0xc041671140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0789898c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0789898c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812361 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021c5b400, {0x63c85b8, 0xc07d21f7d0}, 0xc0342fc450, 0x1, 0xc03b9916d8, 0x519bbc, 0xc05fa47500, 0xc057953080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0664ba900, {0x63c85b8, 0xc07d21f7d0}, 0xc0664ba930, 0xc03b9917d8, 0x30, 0x40, 0xffffffffffffffff, 0xc05fa47500, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021c5b400, {0x63c85b8, 0xc07d21f7d0}, 0xc05fa47440, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc02907c6c0, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07d21f7d0}, 0x6352260, 0x54d3a580) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c42d20, {0x63c85b8, 0xc07d21f7d0}, {0xc05fa47320, {0x174301104e6522ec, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c42d20, {0x63c85b8, 0xc07d21f7d0}, 0xc066c92320, 0xc03b992020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07d21f7d0}, 0x660089, {0xc05fa47320, {0x174301104e6522ec, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c87400, {0x63c85b8, 0xc07d21f7d0}, 0xc0a7625a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e6522ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e6522ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07d21f740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07d21f740}, {0xc088137a28, 0x660089}, 0xc088137a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07d21f740}, {0xc00102e4e0}, 0xc0a7625900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07d21f6e0}, 0xc0a7625900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc08dd8f250, {0x63c85b8, 0xc07d21f650}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc08dd8f250, {0x63c85b8, 0xc07d21f650}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc07d21f650}, {0x4f67ad2, 0x6446ff8}, 0xc002c9efb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219809302 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7eba40, {0x63c85b8, 0xc077f038f0}, 0xc0306169d8, 0x1, 0xc019c090a8, 0x519bbc, 0xc040a8b800, 0xc073574060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077f038f0}, 0x120, 0xc019c09230, 0x40a8b7a0, 0x0, 0x660069, 0xc040a8b800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7eba40, {0x63c85b8, 0xc077f038f0}, 0xc040a8b7a0, {{{{0x0, 0x0}, {0xc00f102120, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077f038f0}, 0x6352410, 0x42a0f2b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02d86de00, {0x63c85b8, 0xc077f038f0}, {0xc040a8b680, {0x1743011064974df9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02d86de00, {0x63c85b8, 0xc077f038f0}, 0xc077d0b540, 0xc019c099f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077f038f0}, 0x660089, {0xc040a8b680, {0x1743011064974df9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e839c00, {0x63c85b8, 0xc077f038f0}, 0xc0604d0b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011064974df9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011064974df9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077f03860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077f03860}, {0xc019c0b3f8, 0x660089}, 0xc019c0b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077f03860}, {0x3eebd269a1f86560}, 0xc0604d0800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077f037d0}, 0xc0604d0800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}, 0xc02d9d7e20, 0xc0ce34f188) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}, 0x3, 0xc05b56e3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc019c0b980}}}, {0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}, 0xc02d9d7e20, 0xc05b56e3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc077f037d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077f037d0}, {0x40, 0xc019c0bb70}, 0xc019c0bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc077f037d0}, {0x4e9d7a0, 0xc0604d0800}, 0xc02d9d7e20, 0xc0ce34f188) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc077f037d0}, 0xc01f9cb080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06cf607e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06cf607e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806905 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc092e22780}, 0xc038af8e70, 0x1, 0xc003dc10a8, 0x519bbc, 0xc05f84fe60, 0xc06224c060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc092e22780}, 0x1e0, 0xc003dc1230, 0x5f84fe00, 0x0, 0x660069, 0xc05f84fe60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc092e22780}, 0xc05f84fe00, {{{{0x0, 0x0}, {0xc058954240, 0x5c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc092e22780}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc092e22780}, {0xc081f8b9e0, {0x174301105328510a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc092e22780}, 0xc055072000, 0xc003dc19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc092e22780}, 0x660089, {0xc081f8b9e0, {0x174301105328510a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc092e22780}, 0xc07447f700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105328510a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105328510a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc092e226f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092e226f0}, {0xc003dc33f8, 0x660089}, 0xc003dc3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc092e226f0}, {0x0}, 0xc07447f500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc092e22660}, 0xc07447f500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}, 0xc08fc48800, 0xc038af91e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}, 0x3, 0xc09071d200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003dc3980}}}, {0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}, 0xc08fc48800, 0xc09071d200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc092e22660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092e22660}, {0x40, 0xc003dc3b70}, 0xc003dc3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc092e22660}, {0x4e9d7a0, 0xc07447f500}, 0xc08fc48800, 0xc038af91e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc092e22660}, 0xc07be2a960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081f8b7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081f8b7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810985 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04e7cb740}, 0xc020019920, 0x1, 0xc05b879f20, 0x519bbc, 0xc063ccc2a0, 0xc059ee7aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc04e7cb740}, 0xc0, 0xc05b87a0a8, 0x63ccc240, 0x0, 0x660069, 0xc063ccc2a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04e7cb740}, 0xc063ccc240, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc04e7cb740}, 0x63524a0, 0x1e651fb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04e7cb740}, {0x0, {0x1743011071abbb37, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04e7cb740}, 0xc06fbc8a00, 0x1) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e7cb740}, 0x660089, {0x0, {0x1743011071abbb37, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04e7cb740}, 0xc007c30f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e7cb6b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e7cb6b0}, {0xc010fde270, 0x660089}, 0xc010fde278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e7cb6b0}, {0xc04e7cb3e0}, 0xc007c30e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e7cb650}, 0xc007c30e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x1743011071abbb37, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc04e7cb5c0}, 0x0, 0xc057174c00) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc04e7cb5c0}, 0x66040f) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc019919860, {0x63c85b8, 0xc04e7cb5c0}, {{0xa8, 0xd0, 0xc8, 0xa9, 0xa1, 0xed, 0x4b, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:922 +0x1b0 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc019919860, {0x63c85b8, 0xc04e7cb5c0}, 0xc05bf72d20, 0xc00aef56a0, 0xc03ec93360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:610 +0xd4a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc04e7cb5c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc04e7cb500}, {{0xc035b90400, 0x24, 0x24}}, 0xc010fe1380) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc019919860, {0x63c85b8, 0xc04e7cb500}, 0xc05bf72d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0301af310, {0x63c85b8, 0xc04e7cb500}, {0x0, {0x1743011071a55d45, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04e7cb500}, 0xc042f32640, 0xc010fe19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e7cb500}, 0x660089, {0x0, {0x1743011071a55d45, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04e7cb500}, 0xc007c30d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011071a55d45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011071a55d45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011071a55d45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e7cb470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e7cb470}, {0xc010fe33f8, 0x660089}, 0xc010fe3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e7cb470}, {0xc0015221e0}, 0xc007c30b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e7cb3e0}, 0xc007c30b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}, 0xc00aef5620, 0xc031af6c90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}, 0x3, 0xc035b90340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010fe3980}}}, {0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}, 0xc00aef5620, 0xc035b90340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e7cb3e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e7cb3e0}, {0x40, 0xc010fe3b70}, 0xc010fe3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e7cb3e0}, {0x4e9d7a0, 0xc007c30b00}, 0xc00aef5620, 0xc031af6c90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e7cb3e0}, 0xc01fa8c540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0748bb200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0748bb200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801376 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc076695500}, 0xc03ed20258, 0x1, 0xc05340b0a8, 0x519bbc, 0xc00aaca4e0, 0xc037722ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc076695500}, 0x120, 0xc05340b230, 0xaaca480, 0x0, 0x660069, 0xc00aaca4e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc076695500}, 0xc00aaca480, {{{{0x0, 0x0}, {0xc04565d320, 0x12}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc076695500}, 0x6352410, 0x1a06e628) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc076695500}, {0xc00aaca240, {0x174301101ae98db8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc076695500}, 0xc07cc6e3c0, 0xc05340b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc076695500}, 0x660089, {0xc00aaca240, {0x174301101ae98db8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc076695500}, 0xc0a3e71900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ae98db8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ae98db8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076695440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076695440}, {0xc05340d3f8, 0x660089}, 0xc05340d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076695440}, {0xc0015221e0}, 0xc0a3e71700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0766953b0}, 0xc0a3e71700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}, 0xc00bdabd80, 0xc042de3ad0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}, 0x3, 0xc0cb6ea8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05340d980}}}, {0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}, 0xc00bdabd80, 0xc0cb6ea8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0766953b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0766953b0}, {0x40, 0xc05340db70}, 0xc05340dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0766953b0}, {0x4e9d7a0, 0xc0a3e71700}, 0xc00bdabd80, 0xc042de3ad0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0766953b0}, 0xc0384812c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00aaca000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00aaca000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812557 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc01fce02c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc08437a000, 0xc041ea7168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01fce02c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc08437a000, {0xc073ca4060, 0x28, 0x2e}, {0xc0856b61e0, 0x28, 0x4a}, 0x0, 0x6f) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc03a6e6de0, {0xc073ca4060, 0x1, 0x4a}, {0xc0856b61e0, 0xc03a6e6000, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc03a6e6de0, {0xc073ca4060, 0x28, 0x2e}, {0xc0856b61e0, 0x28, 0x4a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc03a6e62c0, {0xc0856b61e0, 0x0, 0xc041ea74e0}, 0x121ea30, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc03a6e62c0, {0xc073ca4060, 0x28, 0x4a}, {0xc0856b61e0, 0x27, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc03a6e6000, {0xc0856b61e0, 0x28, 0x4a}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05fead188, {{0xc03a992d20, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07376f700, {{0xc03a992d20, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc026189c00, {0x63c85b8, 0xc056615050}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056615050}, {0x64a6560, 0xc07376f700}, {0xc03a992d20, 0x0, 0x30}, {0x1743011093cf29cc, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056615050}, {0x6466ed8, 0xc05fead180}, {0xc03a992d20, 0x27, 0x30}, {0x1743011093cf29cc, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9a6c0}, {{0x1743011093cf29cc, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011093cf29cc, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046448400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b19c8ecc, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02f354400, {0x63c85b8, 0xc056615050}, 0xc046448400, 0xc048ae3b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f354400, {0x63c85b8, 0xc056615050}, 0xc046448400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011093cf29cc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011093cf29cc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056614fc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614fc0}, {0xc041ead3f8, 0x660089}, 0xc041ead400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056614fc0}, {0xc0015221e0}, 0xc046448200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056614f30}, 0xc046448200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}, 0xc0ccf7fe00, 0xc038161ea8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}, 0x3, 0xc071e55200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc041ead980}}}, {0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}, 0xc0ccf7fe00, 0xc071e55200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056614f30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614f30}, {0x40, 0xc041eadb70}, 0xc041eadab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056614f30}, {0x4e9d7a0, 0xc046448200}, 0xc0ccf7fe00, 0xc038161ea8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056614f30}, 0xc071983380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812212 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc0b9ac1980}, 0xc03b041ce0, 0x1, 0x7080181bc0ae00, 0x37e11d600, 0xc00a3c5920, 0xc007905a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0b9ac1980}, 0x120, 0xc037fb3230, 0xa3c58c0, 0x0, 0x660069, 0xc00a3c5920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc0b9ac1980}, 0xc00a3c58c0, {{{{0x0, 0x0}, {0xc02875a900, 0x35}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0b9ac1980}, 0x6352410, 0x47c52328) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc0b9ac1980}, {0xc00a3c57a0, {0x1743011077a7adfe, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc0b9ac1980}, 0xc030ea7f40, 0xc037fb39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0b9ac1980}, 0x660089, {0xc00a3c57a0, {0x1743011077a7adfe, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc0b9ac1980}, 0xc048c44c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077a7adfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077a7adfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0b9ac18f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b9ac18f0}, {0xc037fb53f8, 0x660089}, 0xc037fb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0b9ac18f0}, {0x36}, 0xc048c44a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0b9ac1860}, 0xc048c44a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}, 0xc01e5172a0, 0xc03b041cb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}, 0x3, 0xc018743ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc037fb5980}}}, {0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}, 0xc01e5172a0, 0xc018743ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0b9ac1860}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b9ac1860}, {0x40, 0xc037fb5b70}, 0xc037fb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0b9ac1860}, {0x4e9d7a0, 0xc048c44a00}, 0xc01e5172a0, 0xc03b041cb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0b9ac1860}, 0xc082ff94a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00a3c5560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00a3c5560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 216823109 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af297f8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0b21c7480, 0xc0175a7000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0b21c7480, {0xc0175a7000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0b21c7480, {0xc0175a7000, 0x735, 0xc03bc27540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc04d763ed0, {0xc0175a7000, 0x7eda79, 0xc0175a7005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01fa8c720, {0xc0175a7000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0615a3860, {0xc0175a7000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc070543e78, {0x631b900, 0xc0615a3860}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc070543c00, {0x7fe42ab59310, 0xc01fa8c720}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc070543c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc070543c00, {0xc0b4c15000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01636b788, {0xc0b4c15000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc059cb2088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc059cb2088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc059cb2660, {0x634aac0, 0xc059cb2088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc059cb2000, 0xc0470d0270, 0xc08bf4742b, 0xc08bf4742d, {0x635bec0, 0xc041ac8ea0}, 0xc000d6ae00, 0xc05fcd3458, 0xc08bf47429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc059cb2000, {0x63c85b8, 0xc06b1451a0}, 0xc0470d0260, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc01dbc3021, 0x8}}, 0x0, 0xc06b145140, 0xc06b145170, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06b1451a0}, {0x643d810, 0xc01fa8c720}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc050facd20}, {0x643d810, 0xc01fa8c720}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc050facd20}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812608 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc04f63b4a0}, 0xc031cfaeb8, 0x1, 0xc03d8df0a8, 0x519bbc, 0xc083a48ae0, 0xc0579f4f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05afa94c0, {0x63c85b8, 0xc04f63b4a0}, 0xc073da27e0, 0xc03d8df1a8, 0x83a48ae0, 0x0, 0x0, 0xc083a48ae0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc04f63b4a0}, 0xc083a48a20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc02f72e000, 0x52}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f63b4a0}, 0x6352260, 0x47c525e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc04f63b4a0}, {0xc083a48900, {0x1743011058606be6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc04f63b4a0}, 0xc05a18b0e0, 0xc03d8df9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f63b4a0}, 0x660089, {0xc083a48900, {0x1743011058606be6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc04f63b4a0}, 0xc0455a3000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058606be6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058606be6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63b410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b410}, {0xc03d8e13f8, 0x660089}, 0xc03d8e1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63b410}, {0x7fe3e21b7048}, 0xc0455a2c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63b260}, 0xc0455a2c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}, 0xc03ebdd4c0, 0xc031cfafc0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}, 0x3, 0xc033d337c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03d8e1980}}}, {0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}, 0xc03ebdd4c0, 0xc033d337c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63b260}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b260}, {0x40, 0xc03d8e1b70}, 0xc03d8e1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63b260}, {0x4e9d7a0, 0xc0455a2c00}, 0xc03ebdd4c0, 0xc031cfafc0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63b260}, 0xc039f0a4e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e065b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e065b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805064 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc0461efaa0}, 0xc05b94fbc0, 0x1, 0x708017dc0ad15e, 0x37e11d600, 0xc0736c0840, 0xc06ec5b380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0461efaa0}, 0x120, 0xc087a95230, 0x736c07e0, 0xc01e743a00, 0x18, 0xc0736c0840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc0461efaa0}, 0xc0736c07e0, {{{{0x0, 0x0}, {0xc056cb4fc0, 0x3a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0461efaa0}, 0x6352410, 0x24ff0370) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc0461efaa0}, {0xc0736c06c0, {0x174301103d913658, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc0461efaa0}, 0xc077402320, 0xc087a959f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0461efaa0}, 0x660089, {0xc0736c06c0, {0x174301103d913658, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0461efaa0}, 0xc089023400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103d913658, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103d913658, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0461efa10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0461efa10}, {0xc087a973f8, 0x660089}, 0xc087a97400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0461efa10}, {0x7fe13d169580}, 0xc089023200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0461ef980}, 0xc089023200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}, 0xc01af61380, 0xc05b94fb90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}, 0x3, 0xc073363a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc087a97980}}}, {0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}, 0xc01af61380, 0xc073363a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0461ef980}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0461ef980}, {0x40, 0xc087a97b70}, 0xc087a97ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}, 0x20301c, 0x20301c) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0461ef980}, {0x4e9d7a0, 0xc089023200}, 0xc01af61380, 0xc05b94fb90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0461ef980}, 0xc014b028a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0736c0360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0736c0360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812796 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 210356930 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0663a7850, 0x135) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc004d99500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc004d99b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc064319300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc064319300, {0x63c8510, 0xc0663a7880}, 0xc0885b9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc027c28890) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0663a7880}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc027c28890) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 205581564 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc04f378650, 0x40b) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0a1a7ca80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0a1a7d0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc054d25900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc054d25900, {0x63c8510, 0xc04f378680}, 0xc04aa79dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0297c02d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc04f378680}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0297c02d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804384 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b728c00, {0x63c85b8, 0xc07c9bad20}, 0xc05221b300, 0xc029a1bea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b728c00, {0x63c85b8, 0xc07c9bad20}, 0xc05221b300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011038884344, 0x0, 0x0}, 0xc041ff2980, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011038884344, 0x0, 0x0}, 0xc041ff2980, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c9bac90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c9bac90}, {0xc0872e13f8, 0x660089}, 0xc0872e1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c9bac90}, {0x0}, 0xc05221b100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c9bac00}, 0xc05221b100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}, 0xc046b43ac0, 0xc03807eb10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}, 0x3, 0xc060a7ce80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0872e1980}}}, {0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}, 0xc046b43ac0, 0xc060a7ce80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c9bac00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c9bac00}, {0x40, 0xc0872e1b70}, 0xc0872e1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}, 0x203018, 0x203018) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c9bac00}, {0x4e9d7a0, 0xc05221b100}, 0xc046b43ac0, 0xc03807eb10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c9bac00}, 0xc01c30a5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0644c4120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0644c4120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809901 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04f383c80}, 0xc06c871818, 0x1, 0xc01b9050a8, 0x519bbc, 0xc085288ae0, 0xc038ab1080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06e334360, {0x63c85b8, 0xc04f383c80}, 0xc07cd11560, 0xc01b9051a8, 0x85288ae0, 0x0, 0x0, 0xc085288ae0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04f383c80}, 0xc085288a20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc087ac0d80, 0x8c}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f383c80}, 0x6352260, 0x1fa2f0c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04f383c80}, {0xc085288900, {0x174301102cc46955, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04f383c80}, 0xc0738b3720, 0xc01b9059f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f383c80}, 0x660089, {0xc085288900, {0x174301102cc46955, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04f383c80}, 0xc06502ae00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc46955, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc46955, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f383bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f383bf0}, {0xc01b9073f8, 0x660089}, 0xc01b907400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f383bf0}, {0x634e480}, 0xc06502ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f383b60}, 0xc06502ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}, 0xc085fd3c20, 0xc0403e7830) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}, 0x3, 0xc05eaecfc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01b907980}}}, {0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}, 0xc085fd3c20, 0xc05eaecfc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f383b60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f383b60}, {0x40, 0xc01b907b70}, 0xc01b907ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f383b60}, {0x4e9d7a0, 0xc06502ac00}, 0xc085fd3c20, 0xc0403e7830) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f383b60}, 0xc01cbd2f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08252c240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08252c240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811613 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0b2c00, {0x63c85b8, 0xc076300480}, 0xc076dfec00, 0xc050934aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc076300480}, 0xc076dfec00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107cc7b567, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107cc7b567, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0763003f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0763003f0}, {0xc00bc0d3f8, 0x660089}, 0xc00bc0d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0763003f0}, {0x7fe3ba7617b8}, 0xc076dfea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076300330}, 0xc076dfea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}, 0xc044a37960, 0xc045705740) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}, 0x3, 0xc0875f4900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00bc0d980}}}, {0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}, 0xc044a37960, 0xc0875f4900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076300330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076300330}, {0x40, 0xc00bc0db70}, 0xc00bc0dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076300330}, {0x4e9d7a0, 0xc076dfea00}, 0xc044a37960, 0xc045705740) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076300330}, 0xc0209e1440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0399a8360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0399a8360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809875 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc062caf920}, 0xc03df03458, 0x1, 0xc0127f30a8, 0x519bbc, 0xc06136a420, 0xc0735e8d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc062caf920}, 0x1e0, 0xc0127f3230, 0x6136a3c0, 0x0, 0x660069, 0xc06136a420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc062caf920}, 0xc06136a3c0, {{{{0x0, 0x0}, {0xc028414900, 0x5e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc062caf920}, 0x63521a0, 0x1b308a28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc062caf920}, {0xc071f57d40, {0x1743011066f2ec44, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc062caf920}, 0xc020ee9360, 0xc0127f39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc062caf920}, 0x660089, {0xc071f57d40, {0x1743011066f2ec44, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc062caf920}, 0xc06570b200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066f2ec44, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066f2ec44, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062caf890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062caf890}, {0xc0127f53f8, 0x660089}, 0xc0127f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062caf890}, {0x0}, 0xc06570b000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062caf800}, 0xc06570b000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}, 0xc04f23d2a0, 0xc0b66536f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}, 0x3, 0xc065e38600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0127f5980}}}, {0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}, 0xc04f23d2a0, 0xc065e38600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062caf800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062caf800}, {0x40, 0xc0127f5b70}, 0xc0127f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062caf800}, {0x4e9d7a0, 0xc06570b000}, 0xc04f23d2a0, 0xc0b66536f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062caf800}, 0xc06d55b200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc081db70e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc081db70e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801548 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc018574800, {0x63c85b8, 0xc02a855e90}, 0xc073962c00, 0xc050934460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018574800, {0x63c85b8, 0xc02a855e90}, 0xc073962c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101cf96b68, 0x0, 0x0}, 0xc04871e730, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101cf96b68, 0x0, 0x0}, 0xc04871e730, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02a855b90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02a855b90}, {0xc07b0d5918, 0x660089}, 0xc07b0d5920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02a855b90}, {0xc00102e4e0}, 0xc073962b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02a855a10}, 0xc073962b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06b84db00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04173b140}, {0xc00dfd36b0, 0x24}, 0x470296ead, 0xc0185ced40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc02c506ba0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04173b140}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 212143039 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b25e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc065051800, 0xc034d8b000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc065051800, {0xc034d8b000, 0x746, 0x746}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc065051800, {0xc034d8b000, 0x0, 0xc053b72bc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01ecbb568, {0xc034d8b000, 0x100, 0xff}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0208f5620, {0xc034d8b000, 0x4bf403, 0x746}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0ccf026d8, {0xc034d8b000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc033e9b3f8, {0x631b900, 0xc0ccf026d8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc033e9b180, {0x7fe42ab59310, 0xc0208f5620}, 0x746) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc033e9b180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc033e9b180, {0xc0944fa000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03f147950, {0xc0944fa000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03df89588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03df89588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03df89b60, {0x634aac0, 0xc03df89588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03df89500, 0xc058511a80, 0xc053b7342b, 0xc053b7342d, {0x635bec0, 0xc03b721950}, 0xc000d6ae00, 0xc04ba24924, 0xc053b73429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03df89500, {0x63c85b8, 0xc02e64cf60}, 0xc058511a70, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc089e91009, 0x18}}, 0x0, 0xc02e64ce70, 0xc02e64cf00, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02e64cf60}, {0x643d810, 0xc0208f5620}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc041a9bcb0}, {0x643d810, 0xc0208f5620}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc041a9bcb0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 210356852 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa8e30, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc065cc9000, 0xc0388f3000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc065cc9000, {0xc0388f3000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc065cc9000, {0xc0388f3000, 0x5, 0xc058192bc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0204d39f8, {0xc0388f3000, 0x40, 0x6}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc032853aa0, {0xc0388f3000, 0xc076c92c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03c545c20, {0xc0388f3000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0609705f8, {0x631b900, 0xc03c545c20}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc060970380, {0x7fe42ab59310, 0xc032853aa0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc060970380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc060970380, {0xc0293d5000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc045c2c330, {0xc0293d5000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc004d99588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc004d99588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc004d99b60, {0x634aac0, 0xc004d99588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc004d99500, 0xc027c28870, 0xc05819342b, 0xc05819342d, {0x635bec0, 0xc00e3d9050}, 0xc000d6ae00, 0xc03d46e508, 0xc058193429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc004d99500, {0x63c85b8, 0xc04b590810}, 0xc027c28860, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc02207b025, 0x8}}, 0x0, 0xc04b5906f0, 0xc04b590780, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04b590810}, {0x643d810, 0xc032853aa0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0571eba70}, {0x643d810, 0xc032853aa0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0571eba70}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803709 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaffb3ec0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05b944e80, 0xc04b861800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05b944e80, {0xc04b861800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05b944e80, {0xc04b861800, 0x739, 0xc08eca9900}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc045bf49c8, {0xc04b861800, 0x7eda79, 0xc04b861805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc030123740, {0xc04b861800, 0x4a7cec0, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc073113cc8, {0xc04b861800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04d778978, {0x631b900, 0xc073113cc8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04d778700, {0x7fe42ab59310, 0xc030123740}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04d778700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04d778700, {0xc05add4000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04b9a7860, {0xc05add4000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04467b588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04467b588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04467bb60, {0x634aac0, 0xc04467b588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04467b500, 0xc0534b1f20, 0xc096c0142b, 0xc096c0142d, {0x635bec0, 0xc0439fc240}, 0xc000d6ae00, 0xc083b59444, 0xc096c01429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04467b500, {0x63c85b8, 0xc07c7b8e10}, 0xc0534b1f10, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc032892043, 0xb}}, 0x0, 0xc07c7b8db0, 0xc07c7b8de0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc07c7b8e10}, {0x643d810, 0xc030123740}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc074dbcd20}, {0x643d810, 0xc030123740}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc074dbcd20}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807809 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b7643c0, {0x63c85b8, 0xc0373232c0}, 0xc02c5ed1d0, 0x1, 0x0, 0x1, 0xc07cab84e0, 0xc07854c840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0373232c0}, 0x120, 0xc008258728, 0x7cab8480, 0x0, 0x660069, 0xc07cab84e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b7643c0, {0x63c85b8, 0xc0373232c0}, 0xc07cab8480, {{{{0x0, 0x0}, {0xc0aa242900, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0373232c0}, 0x6352410, 0x2ad6f770) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b6f5180, {0x63c85b8, 0xc0373232c0}, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b6f5180, {0x63c85b8, 0xc0373232c0}, 0xc03b364fa0, 0xc07c6c4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0373232c0}, 0x660089, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b772000, {0x63c85b8, 0xc0373232c0}, 0xc01fb36700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc037323200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037323200}, {0xc0936368f0, 0x660089}, 0xc0936368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc037323200}, {0xc00102e4e0}, 0xc01fb36600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0373231a0}, 0xc01fb36600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc037322690}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802824 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b26c000, {0x63c85b8, 0xc047e331a0}, 0xc046fdcc00, 0xc039b4af00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc047e331a0}, 0xc046fdcc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110256038b7, 0x0, 0x0}, 0xc03060b080, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110256038b7, 0x0, 0x0}, 0xc03060b080, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e33110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e33110}, {0xc064b953f8, 0x660089}, 0xc064b95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e33110}, {0x0}, 0xc046fdca00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e33050}, 0xc046fdca00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}, 0xc0968d07c0, 0xc06316fd10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}, 0x3, 0xc056d17800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc064b95980}}}, {0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}, 0xc0968d07c0, 0xc056d17800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047e33050}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e33050}, {0x40, 0xc064b95b70}, 0xc064b95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047e33050}, {0x4e9d7a0, 0xc046fdca00}, 0xc0968d07c0, 0xc06316fd10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047e33050}, 0xc0974e2900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0478e3e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0478e3e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219798714 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc07baa0548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc07baa05b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc094e216f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0a56b4e00, {0xc071a8b3b0, 0x2b, 0x6c}, {0xc024361800, 0x2b, 0x76}, 0x1, 0x16) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc094e21588, {0xc071a8b3b0, 0x1, 0x76}, {0xc024361800, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc094e21588, {0xc071a8b3b0, 0x2b, 0x6c}, {0xc024361800, 0x2b, 0x76}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc094e20cc0, {0xc024361800, 0x76, 0x6c}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc094e20cc0, {0xc071a8b3b0, 0x2b, 0xc024361800}, {0xc024361800, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc094e20a00, {0xc024361800, 0x2b, 0x76}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc08b50c008, {{0xc0476e7350, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06ab87000, {{0xc0476e7350, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0501d0400, {0x63c85b8, 0xc063c1c2d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc063c1c2d0}, {0x64a6560, 0xc06ab87000}, {0xc0476e7350, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc063c1c2d0}, {0x6466ed8, 0xc08b50c000}, {0xc0476e7350, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9a288}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc058056300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020545400, {0x63c85b8, 0xc063c1c2d0}, 0xc058056300, 0xc02ec239a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020545400, {0x63c85b8, 0xc063c1c2d0}, 0xc058056300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc063c1c240}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063c1c240}, {0xc09365a8f0, 0x660089}, 0xc09365a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc063c1c240}, {0xc00102e4e0}, 0xc058056200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc063c1c1e0}, 0xc058056200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03c4c6960}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806726 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc011ca7400, {0x63c85b8, 0xc08d8ed530}, 0xc043b8e300, 0xc068880140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc011ca7400, {0x63c85b8, 0xc08d8ed530}, 0xc043b8e300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105093de84, 0x0, 0x0}, 0xc08dbd20c0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105093de84, 0x0, 0x0}, 0xc08dbd20c0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08d8ed4a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d8ed4a0}, {0xc0c14048f0, 0x660089}, 0xc0c14048f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08d8ed4a0}, {0xc00102e4e0}, 0xc043b8e100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08d8ed440}, 0xc043b8e100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc08d3dd8f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812590 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812805 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812382 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc08aff01d0, {0x63c85b8, 0xc0738945d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0738945d0}, {0x4f67ad2, 0x6446ff8}, 0xc0385e6fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219775818 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0571e7490, 0x2a) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0965b6a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0965b70e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05f449900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05f449900, {0x63c8510, 0xc0571e7500}, 0xc0bbce9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01ee43b10) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0571e7500}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01ee43b10) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 207017524 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc025eab390, 0x88a) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc084628000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc084628660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0688b4000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0688b4000, {0x63c8510, 0xc025eab3c0}, 0xc08a993dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02ef92530) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc025eab3c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02ef92530) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802127 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc017b8b000, {0x63c85b8, 0xc027313c50}, 0xc070039e00, 0xc0c59ae5a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017b8b000, {0x63c85b8, 0xc027313c50}, 0xc070039e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011021626474, 0x0, 0x0}, 0xc024e29290, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011021626474, 0x0, 0x0}, 0xc024e29290, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027313bc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027313bc0}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027313bc0}, {0xc00102e4e0}, 0xc070039d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027313b60}, 0xc070039d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc088a61320}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc027313a10}, {0xc00dfd36b0, 0x24}, 0x46c7c4e56, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc08ccac680, 0xc00e4eac78}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc027313a10}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804941 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc03897b320}, 0xc08afc6c48, 0x1, 0xc07baad0a8, 0x519bbc, 0xc055b21320, 0xc0552e9e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03897b320}, 0x51cdcd8, 0xc07baad230, 0x7baad138, 0x4c166a, 0x18, 0xc055b21320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc03897b320}, 0xc055b212c0, {{{{0x0, 0x0}, {0xc057528fc0, 0x51}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03897b320}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc03897b320}, {0xc06f9cea20, {0x174301103d365dfe, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc03897b320}, 0xc030cce640, 0xc07baad9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03897b320}, 0x660089, {0xc06f9cea20, {0x174301103d365dfe, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc03897b320}, 0xc06f10b900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103d365dfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103d365dfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03897b290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03897b290}, {0xc07baaf3f8, 0x660089}, 0xc07baaf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03897b290}, {0xc0015221e0}, 0xc06f10b700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03897b200}, 0xc06f10b700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}, 0xc060512620, 0xc08afc73e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}, 0x3, 0xc016ae2d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07baaf980}}}, {0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}, 0xc060512620, 0xc016ae2d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03897b200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03897b200}, {0x40, 0xc07baafb70}, 0xc07baafab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}, 0x203005, 0x203005) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03897b200}, {0x4e9d7a0, 0xc06f10b700}, 0xc060512620, 0xc08afc73e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03897b200}, 0xc064bb1560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06f9ce900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06f9ce900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800938 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc030c04500, {0x63c85b8, 0xc044cca270}, 0xc047cefba8, 0x1, 0xc04c8050a8, 0x519bbc, 0xc032c7c060, 0xc0297d36e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc044cca270}, 0x1e0, 0xc04c805230, 0x32c7c000, 0x0, 0x660069, 0xc032c7c060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc030c04500, {0x63c85b8, 0xc044cca270}, 0xc032c7c000, {{{{0x0, 0x0}, {0xc0257707e0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc044cca270}, 0x28, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030bc4d70, {0x63c85b8, 0xc044cca270}, {0xc05bae50e0, {0x1743011015275d8f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030bc4d70, {0x63c85b8, 0xc044cca270}, 0xc076f61a40, 0xc04c8059f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044cca270}, 0x660089, {0xc05bae50e0, {0x1743011015275d8f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030c09000, {0x63c85b8, 0xc044cca270}, 0xc04b06c000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011015275d8f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011015275d8f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044cca1e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044cca1e0}, {0xc04c8073f8, 0x660089}, 0xc04c807400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044cca1e0}, {0x32}, 0xc027fbac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044cca0c0}, 0xc027fbac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}, 0xc0282b6b60, 0xc049505500) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}, 0x3, 0xc00e76e4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04c807980}}}, {0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}, 0xc0282b6b60, 0xc00e76e4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044cca0c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044cca0c0}, {0x40, 0xc04c807b70}, 0xc04c807ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044cca0c0}, {0x4e9d7a0, 0xc027fbac00}, 0xc0282b6b60, 0xc049505500) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044cca0c0}, 0xc02d69fc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05bae4fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05bae4fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803923 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021c06000, {0x63c85b8, 0xc0325f4f00}, 0xc070567200, 0xc0480839a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c06000, {0x63c85b8, 0xc0325f4f00}, 0xc070567200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011016897f20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011016897f20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0325f4cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0325f4cf0}, {0xc0764313f8, 0x660089}, 0xc076431400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0325f4cf0}, {0xc04b7db0e0}, 0xc070567000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0325f4c30}, 0xc070567000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}, 0xc085620c40, 0xc0b43aca80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}, 0x3, 0xc06d188980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076431980}}}, {0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}, 0xc085620c40, 0xc06d188980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0325f4c30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0325f4c30}, {0x40, 0xc076431b70}, 0xc076431ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0325f4c30}, {0x4e9d7a0, 0xc070567000}, 0xc085620c40, 0xc0b43aca80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0325f4c30}, 0xc03552cfc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04c05f680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04c05f680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808728 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6000, {0x63c85b8, 0xc030f8f590}, 0xc05294b140, 0x1, 0xc0130e10a8, 0x519bbc, 0xc0848e1b60, 0xc04dd67140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc030f8f590}, 0x120, 0xc0130e1230, 0x848e1b00, 0x0, 0x660069, 0xc0848e1b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6000, {0x63c85b8, 0xc030f8f590}, 0xc0848e1b00, {{{{0x0, 0x0}, {0xc0154f66c0, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc030f8f590}, 0x63526b0, 0x46e76340) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddb30, {0x63c85b8, 0xc030f8f590}, {0xc0848e19e0, {0x174301105fffbf3b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc030f8f590}, 0xc047208780, 0xc0130e19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc030f8f590}, 0x660089, {0xc0848e19e0, {0x174301105fffbf3b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc030f8f590}, 0xc06c79b000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105fffbf3b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105fffbf3b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc030f8f1a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030f8f1a0}, {0xc0130e33f8, 0x660089}, 0xc0130e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc030f8f1a0}, {0x7fe338069180}, 0xc06c79ae00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc030f8ef30}, 0xc06c79ae00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}, 0xc017f13fc0, 0xc050aa2db0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}, 0x3, 0xc01033bf40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0130e3980}}}, {0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}, 0xc017f13fc0, 0xc01033bf40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc030f8ef30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030f8ef30}, {0x40, 0xc0130e3b70}, 0xc0130e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc030f8ef30}, {0x4e9d7a0, 0xc06c79ae00}, 0xc017f13fc0, 0xc050aa2db0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc030f8ef30}, 0xc03e822480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0848e17a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0848e17a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812210 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc06b42d4d0}, 0xc045704b58, 0x1, 0xc036e290a8, 0x519bbc, 0xc044222a80, 0xc0518bede0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06b42d4d0}, 0x120, 0xc036e29230, 0x44222a20, 0x0, 0x660069, 0xc044222a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc06b42d4d0}, 0xc044222a20, {{{{0x0, 0x0}, {0xc06684e6c0, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06b42d4d0}, 0x6352410, 0x47c52220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc06b42d4d0}, {0xc044222900, {0x174301107dc2e172, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc06b42d4d0}, 0xc0a64a79a0, 0xc036e299f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b42d4d0}, 0x660089, {0xc044222900, {0x174301107dc2e172, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc06b42d4d0}, 0xc05bd2d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107dc2e172, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107dc2e172, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b42d440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b42d440}, {0xc036e2b3f8, 0x660089}, 0xc036e2b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b42d440}, {0x7fe3b8bce060}, 0xc05bd2d000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b42d3b0}, 0xc05bd2d000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}, 0xc01e516a00, 0xc03b040dc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}, 0x3, 0xc018742cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc036e2b980}}}, {0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}, 0xc01e516a00, 0xc018742cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b42d3b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b42d3b0}, {0x40, 0xc036e2bb70}, 0xc036e2bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b42d3b0}, {0x4e9d7a0, 0xc05bd2d000}, 0xc01e516a00, 0xc03b040dc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b42d3b0}, 0xc08c159ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0442227e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0442227e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812803 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802678 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x6b) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x100, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0911160f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0b4248380, {0xc0581dc1e0, 0x32, 0x60}, {0xc016b47a00, 0x32, 0x72}, 0x1, 0x60) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091115f88, {0xc0581dc1e0, 0x1, 0x72}, {0xc016b47a00, 0x2b1, 0xc0775ca790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091115f88, {0xc0581dc1e0, 0x32, 0x60}, {0xc016b47a00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0911156c0, {0xc016b47a00, 0x72, 0x60}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0911156c0, {0xc0581dc1e0, 0x72, 0xc016b47a00}, {0xc016b47a00, 0xc079c26bd0, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091115400, {0xc016b47a00, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc086b86ea8, {{0xc079c26bd0, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc089023800, {{0xc05c0538c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc090bd6000, {0x63c85b8, 0xc06ff87470}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06ff87470}, {0x64a6560, 0xc089023800}, {0xc05c0538c0, 0x1e, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06ff87470}, {0x6466ed8, 0xc086b86c00}, {0xc05c0538c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc044617b60}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc050d52e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4e800, {0x63c85b8, 0xc06ff87470}, 0xc050d52e00, 0xc02f165900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4e800, {0x63c85b8, 0xc06ff87470}, 0xc050d52e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ff873e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ff873e0}, {0xc0936988f0, 0x660089}, 0xc0936988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ff873e0}, {0xc00102e4e0}, 0xc050d52d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06ff87380}, 0xc050d52d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220e40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800939 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018383400, {0x63c85b8, 0xc0584430b0}, 0xc02a0b29f0, 0x1, 0x0, 0x1, 0xc030360600, 0xc0609d44e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0584430b0}, 0x120, 0xc04ce26728, 0x303605a0, 0x0, 0x660069, 0xc030360600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018383400, {0x63c85b8, 0xc0584430b0}, 0xc0303605a0, {{{{0x0, 0x0}, {0xc09913bd40, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0584430b0}, 0x6352410, 0x1d9d0d78) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0181dfb30, {0x63c85b8, 0xc0584430b0}, {0xc05bae5e60, {0x174301101615db41, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0181dfb30, {0x63c85b8, 0xc0584430b0}, 0xc037d43cc0, 0xc0768eaee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0584430b0}, 0x660089, {0xc05bae5e60, {0x174301101615db41, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0184f7c00, {0x63c85b8, 0xc0584430b0}, 0xc04be37c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101615db41, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101615db41, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc058443020}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058443020}, {0xc00310a8f0, 0x660089}, 0xc00310a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc058443020}, {0xc00102e4e0}, 0xc04be37a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc058442fc0}, 0xc04be37a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc044ccac30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812279 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016d6d2c0, {0x63c85b8, 0xc074c0e210}, 0xc017f113f8, 0x1, 0xc0365bd0a8, 0x519bbc, 0xc09fd4dc80, 0xc024ed0ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc074c0e210}, 0x120, 0xc0365bd230, 0x9fd4dc20, 0x0, 0x660069, 0xc09fd4dc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016d6d2c0, {0x63c85b8, 0xc074c0e210}, 0xc09fd4dc20, {{{{0x0, 0x0}, {0xc07455c6c0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc074c0e210}, 0x6352410, 0x3abf6368) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce3180, {0x63c85b8, 0xc074c0e210}, {0xc09fd4db00, {0x174301107e50a6b9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce3180, {0x63c85b8, 0xc074c0e210}, 0xc055b234a0, 0xc0365bd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc074c0e210}, 0x660089, {0xc09fd4db00, {0x174301107e50a6b9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016efac00, {0x63c85b8, 0xc074c0e210}, 0xc07d476700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074c0e120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074c0e120}, {0xc0365bf3f8, 0x660089}, 0xc0365bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074c0e120}, {0x7fe3262c9fd8}, 0xc07d476500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074c0e090}, 0xc07d476500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}, 0xc01c0228c0, 0xc017f11728) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}, 0x3, 0xc06541df00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0365bf980}}}, {0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}, 0xc01c0228c0, 0xc06541df00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc074c0e090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074c0e090}, {0x40, 0xc0365bfb70}, 0xc0365bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc074c0e090}, {0x4e9d7a0, 0xc07d476500}, 0xc01c0228c0, 0xc017f11728) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc074c0e090}, 0xc02d3a7140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09fd4d9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09fd4d9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219141872 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc04ec3d310, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc040d54ea0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.newHTTP2Client.func3() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:415 +0x65 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:413 +0x1fa5 goroutine 219802260 [runnable]: sync.runtime_Semrelease(0x1, 0x0, 0x0) GOROOT/src/runtime/sema.go:66 +0x19 sync.(*Mutex).unlockSlow(0x0, 0x0) GOROOT/src/sync/mutex.go:224 +0x65 sync.(*Mutex).Unlock(...) GOROOT/src/sync/mutex.go:190 sync.(*RWMutex).Unlock(0xc00074b2c0) GOROOT/src/sync/rwmutex.go:149 +0x9b github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074b2b0, 0x11869e5, 0x7fe3761c5fc0, 0x7f72, 0x7fd754d60ec0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:206 +0x2fc github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc025717b00, 0x7fe3761c5f01, 0x7fe3761c5fe0, 0x4350, 0x4355) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc025849b80, {0x7da0d14, 0x4350}, 0x0, 0xc078d1dd78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc078d1db00, {0xffffffffffffff15, 0xc09866a5c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc078d1db00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc078d1db00, {0xc03938c0e0, 0x2b, 0x6d}, 0x13, 0x35) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc078d1db00, {0xc064aee960, 0x2b, 0x6d}, {0xc03938c0e0, 0x51c9860, 0xc03938c0e0}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc078d1db00, {0xc064aee960, 0x2b, 0xf0}, {0xc03938c0e0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0becdc398, {0xc064aee960, 0x2b, 0xf0}, {0xc03938c0e0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0becdacc0, {0xc03938c0e0, 0x6d, 0xf0}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0becdacc0, {0xc064aee960, 0x2b, 0xc03938c0e0}, {0xc03938c0e0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0becdaa00, {0xc03938c0e0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05a32d188, {{0xc0421a0f60, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc049f1e900, {{0xc0421a0f60, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc045cf2400, {0x63c85b8, 0xc0672c0870}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0672c0870}, {0x64a6560, 0xc049f1e900}, {0xc0421a0f60, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0672c0870}, {0x6466ed8, 0xc05a32d180}, {0xc0421a0f60, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c5ec510}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04e7de100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020edc000, {0x63c85b8, 0xc0672c0870}, 0xc04e7de100, 0xc050da1360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020edc000, {0x63c85b8, 0xc0672c0870}, 0xc04e7de100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0672c07e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0672c07e0}, {0xc001c988f0, 0x660089}, 0xc001c988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0672c07e0}, {0x24}, 0xc04e7de000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0672c0780}, 0xc04e7de000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0bb639470}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806542 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc087191f50, 0x8) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc04dd03500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc04dd03b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08bc48600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08bc48600, {0x63c8510, 0xc087191f80}, 0xc07685ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05aa488d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc087191f80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05aa488d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810038 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0201e3b80, {0x63c85b8, 0xc01d95fce0}, 0xc037e1ef48, 0x1, 0x708018079ccb7f, 0x37e11d600, 0xc040c915c0, 0xc03a3f22a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01d95fce0}, 0x120, 0xc01a295230, 0x40c91560, 0x0, 0x660069, 0xc040c915c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0201e3b80, {0x63c85b8, 0xc01d95fce0}, 0xc040c91560, {{{{0x0, 0x0}, {0xc0435a0c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01d95fce0}, 0x63526b0, 0x43a289e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01f8c5950, {0x63c85b8, 0xc01d95fce0}, {0xc040c91440, {0x174301106af0b5db, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01f8c5950, {0x63c85b8, 0xc01d95fce0}, 0xc06ba58d20, 0xc01a2959f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01d95fce0}, 0x660089, {0xc040c91440, {0x174301106af0b5db, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020435400, {0x63c85b8, 0xc01d95fce0}, 0xc0855f4c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106af0b5db, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106af0b5db, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01d95fc20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d95fc20}, {0xc01a2973f8, 0x660089}, 0xc01a297400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01d95fc20}, {0x7fe423f95670}, 0xc0855f4a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01d95fb90}, 0xc0855f4a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}, 0xc06e07e200, 0xc037e1ef18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}, 0x3, 0xc039b55e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a297980}}}, {0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}, 0xc06e07e200, 0xc039b55e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01d95fb90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d95fb90}, {0x40, 0xc01a297b70}, 0xc01a297ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01d95fb90}, {0x4e9d7a0, 0xc0855f4a00}, 0xc06e07e200, 0xc037e1ef18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01d95fb90}, 0xc035ffc5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc040c91200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc040c91200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808568 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020c29180, {0x63c85b8, 0xc034d30cc0}, 0xc05f254840, 0x1, 0xc00a8bd6d8, 0x519bbc, 0xc0666592c0, 0xc0472900c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0b646eea0, {0x63c85b8, 0xc034d30cc0}, 0xc0666592c0, 0xc00a8bd7d8, 0x44, 0x60, 0x0, 0xc0666592c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020c29180, {0x63c85b8, 0xc034d30cc0}, 0xc066659200, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0899886c0, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc034d30cc0}, 0x6352260, 0x6bab2a30) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc020a61630, {0x63c85b8, 0xc034d30cc0}, {0xc0666590e0, {0x17430110234b556f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc020a61630, {0x63c85b8, 0xc034d30cc0}, 0xc0b68cabe0, 0xc00a8be020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc034d30cc0}, 0x660089, {0xc0666590e0, {0x17430110234b556f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c61400, {0x63c85b8, 0xc034d30cc0}, 0xc0359a1400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110234b556f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110234b556f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc034d30ba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc034d30ba0}, {0xc07d139a28, 0x660089}, 0xc07d139a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc034d30ba0}, {0xc00102e4e0}, 0xc0359a1300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc034d30b40}, 0xc0359a1300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc051ce38d0, {0x63c85b8, 0xc034d30ab0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc051ce38d0, {0x63c85b8, 0xc034d30ab0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc034d30ab0}, {0x4f67ad2, 0x6446ff8}, 0xc03463bfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219791853 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc08a144548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc08a1445b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc092e005f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0518f3500, {0xc0195e9d00, 0x2b, 0x77}, {0xc010e4fab0, 0x2b, 0x6e}, 0x1, 0x5) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092e00488, {0xc0195e9d00, 0x1, 0x6e}, {0xc010e4fab0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092e00488, {0xc0195e9d00, 0x2b, 0x77}, {0xc010e4fab0, 0x2b, 0x6e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092dffbc0, {0xc010e4fab0, 0x6e, 0x77}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092dffbc0, {0xc0195e9d00, 0x2b, 0xc010e4fab0}, {0xc010e4fab0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092dff900, {0xc010e4fab0, 0x2b, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05c90f188, {{0xc06b620b40, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc059d43200, {{0xc06b620b40, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0574b4400, {0x63c85b8, 0xc053ec6f30}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc053ec6f30}, {0x64a6560, 0xc059d43200}, {0xc06b620b40, 0x20, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc053ec6f30}, {0x6466ed8, 0xc05c90f180}, {0xc06b620b40, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc068a02780}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01bffb600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f82e800, {0x63c85b8, 0xc053ec6f30}, 0xc01bffb600, 0xc042341680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f82e800, {0x63c85b8, 0xc053ec6f30}, 0xc01bffb600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053ec6de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053ec6de0}, {0xc001c988f0, 0x660089}, 0xc001c988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053ec6de0}, {0xc00102e4e0}, 0xc01bffb500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053ec6d50}, 0xc01bffb500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc053ec6a20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219768479 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc08e5160d0, 0x2f) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc05ebdaa80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc05ebdb0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03da9e600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03da9e600, {0x63c8510, 0xc08e516100}, 0xc0709c5dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07bf1fd60) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc08e516100}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07bf1fd60) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812564 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802819 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01edcb400, {0x63c85b8, 0xc030dc39e0}, 0xc0a38f9600, 0xc0677dc960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc030dc39e0}, 0xc0a38f9600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024dd7ee7, 0x0, 0x0}, 0xc0572a9fc0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024dd7ee7, 0x0, 0x0}, 0xc0572a9fc0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc030dc35c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030dc35c0}, {0xc05f5e93f8, 0x660089}, 0xc05f5e9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc030dc35c0}, {0x5f6e6f697461636f}, 0xc0a38f9400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc030dc34d0}, 0xc0a38f9400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}, 0xc0135d8040, 0xc042d4b398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}, 0x3, 0xc05511ae00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05f5e9980}}}, {0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}, 0xc0135d8040, 0xc05511ae00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc030dc34d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030dc34d0}, {0x40, 0xc05f5e9b70}, 0xc05f5e9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc030dc34d0}, {0x4e9d7a0, 0xc0a38f9400}, 0xc0135d8040, 0xc042d4b398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc030dc34d0}, 0xc030803860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc077283d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc077283d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812127 [runnable]: sync.runtime_SemacquireMutex(0x2ae2a47, 0x40, 0x7fe3567feb60) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x459e98, 0x7fe3b9841000, 0xc032e4ee78) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc06123b1a0, 0x9baf000, 0xc032e4eeb0, 0x4bb045) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7248b00, 0x2da4}, 0x0, 0xc084b8c6f8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc084b8c480, {0xffffffffffffff15, 0xc032e4f040}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc084b8c480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc084b8c480, {0xc037dc0960, 0x2e, 0x52}, 0x13, 0x13) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekGE(0xc084b8c480, {0xc037dc0960, 0x2e, 0x52}, 0xe0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:513 +0xaf github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekGE(0xc084b8c480, {0xc037dc0960, 0x2e, 0x52}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1317 +0x29d github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc092a0be98, {0xc037dc0960, 0x2e, 0x52}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:409 +0xd6 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092a0a7c0, {0xc037dc0960, 0x60, 0x52}, 0x2e, 0x2e) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc092a0a7c0, {0xc037dc0960, 0x2e, 0xc024f32c60}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc092a0a500, {0xc037dc0960, 0x2e, 0x52}, {0xc04c825320, 0x30, 0x60}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc074d81478, {{0xc09dc9bb00, 0x2d, 0xc0}, {0x0, 0x0, 0x0}}, {0xc04c825320, 0x2f, 0x60}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:334 +0x173 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0491e7c00, {{0xc08bcae1e0, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc00ff61000, {{0xc08bcae1e0, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc00ff61000, {0x63c85b8, 0xc01fd4bec0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc08bcae210, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.MVCCScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc08bcae210, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2504 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc057247860}, {{0x174301107e21cca6, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:62 +0x41b github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301107e21cca6, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0503f3400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174301109bef31a6, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e096000, {0x63c85b8, 0xc01fd4bec0}, 0xc0503f3400, 0xc02198dea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e096000, {0x63c85b8, 0xc01fd4bec0}, 0xc0503f3400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e21cca6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e21cca6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fd4be30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fd4be30}, {0xc032e553f8, 0x660089}, 0xc032e55400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fd4be30}, {0x7fe13d1804c0}, 0xc0503f3200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fd4bda0}, 0xc0503f3200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}, 0xc01c022080, 0xc017f10a08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}, 0x3, 0xc06541ccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032e55980}}}, {0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}, 0xc01c022080, 0xc06541ccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01fd4bda0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fd4bda0}, {0x40, 0xc032e55b08}, 0xc032e55ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01fd4bda0}, {0x4e9d7a0, 0xc0503f3200}, 0xc01c022080, 0xc017f10a08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01fd4bda0}, 0xc0313a5bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc01cd10c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc01cd10c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 207017523 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc025eaa790, 0x314) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc084603500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc084603b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00bf26c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00bf26c00, {0x63c8510, 0xc025eaa7c0}, 0xc08ceaddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc034d91850) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc025eaa7c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc034d91850) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802042 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc05304c540}, 0xc0202bce00, 0xc03c3eeb40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc05304c540}, 0xc0202bce00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110215415c6, 0x0, 0x0}, 0xc06176d8e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110215415c6, 0x0, 0x0}, 0xc06176d8e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05304c4b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05304c4b0}, {0xc056af33f8, 0x660089}, 0xc056af3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05304c4b0}, {0x5f4c1f1ebaa48cfb}, 0xc0202bcb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05304c420}, 0xc0202bcb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}, 0xc059bcf280, 0xc06af88198) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}, 0x3, 0xc054f23c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc056af3980}}}, {0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}, 0xc059bcf280, 0xc054f23c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05304c420}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05304c420}, {0x40, 0xc056af3b70}, 0xc056af3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05304c420}, {0x4e9d7a0, 0xc0202bcb00}, 0xc059bcf280, 0xc06af88198) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05304c420}, 0xc04fea9860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc056d28120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc056d28120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807403 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc02c73da10}, 0xc092036e00, 0xc0a64a7900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc02c73da10}, 0xc092036e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110570237ea, 0x0, 0x0}, 0xc0907f4420, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110570237ea, 0x0, 0x0}, 0xc0907f4420, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02c73d860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c73d860}, {0xc078183a08, 0x660089}, 0xc078183a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02c73d860}, {0xc00102e4e0}, 0xc092036d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02c73d770}, 0xc092036d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc02c73d590}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc02c73d590}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807758 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021dee500, {0x63c85b8, 0xc0547071a0}, 0xc08aa69c98, 0x1, 0xc00e12f0a8, 0x519bbc, 0xc044b78a20, 0xc064c5a720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc0547071a0}, 0xc0, 0xc00e12f230, 0x44b789c0, 0x0, 0x660069, 0xc044b78a20, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021dee500, {0x63c85b8, 0xc0547071a0}, 0xc044b789c0, {{{{0x0, 0x0}, {0xc00f103440, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0547071a0}, 0x6352410, 0x324309a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3d4a0, {0x63c85b8, 0xc0547071a0}, {0x0, {0x1743011059b382ea, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3d4a0, {0x63c85b8, 0xc0547071a0}, 0xc0777412c0, 0xc00e12f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0547071a0}, 0x660089, {0x0, {0x1743011059b382ea, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021de4800, {0x63c85b8, 0xc0547071a0}, 0xc0537fd700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011059b382ea, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011059b382ea, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011059b382ea, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc054707110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054707110}, {0xc00e1313f8, 0x660089}, 0xc00e131400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc054707110}, {0x634e480}, 0xc0537fd500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc054707080}, 0xc0537fd500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}, 0xc05858bae0, 0xc037911fc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}, 0x3, 0xc074eaccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e131980}}}, {0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}, 0xc05858bae0, 0xc074eaccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc054707080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054707080}, {0x40, 0xc00e131b70}, 0xc00e131ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc054707080}, {0x4e9d7a0, 0xc0537fd500}, 0xc05858bae0, 0xc037911fc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc054707080}, 0xc09df09ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc073db3440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc073db3440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219719368 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc067b1dc50, 0x59) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc05646ea80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc05646f0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05ae9d300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05ae9d300, {0x63c8510, 0xc067b1dc80}, 0xc087e5fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0626c7110) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc067b1dc80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0626c7110) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219793595 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc078cea548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc078cea5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0931c5bf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc063fc2a80, {0xc0156e2f50, 0x2b, 0x68}, {0xc00d1ba700, 0x2b, 0x6d}, 0x1, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0931c5a88, {0xc0156e2f50, 0x1, 0x6d}, {0xc00d1ba700, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0931c5a88, {0xc0156e2f50, 0x2b, 0x68}, {0xc00d1ba700, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0931c51c0, {0xc00d1ba700, 0x6d, 0x68}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0931c51c0, {0xc0156e2f50, 0x2b, 0xc00d1ba700}, {0xc00d1ba700, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0931c4f00, {0xc00d1ba700, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0616c1708, {{0xc0707af290, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc045890a00, {{0xc0707af290, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ae7f400, {0x63c85b8, 0xc052fb7e60}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc052fb7e60}, {0x64a6560, 0xc045890a00}, {0xc0707af290, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc052fb7e60}, {0x6466ed8, 0xc0616c1700}, {0xc0707af290, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc075d0e078}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0a3a98a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020010800, {0x63c85b8, 0xc052fb7e60}, 0xc0a3a98a00, 0xc0846f52c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020010800, {0x63c85b8, 0xc052fb7e60}, 0xc0a3a98a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc052fb7da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052fb7da0}, {0xc0032fe8f0, 0x660089}, 0xc0032fe8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc052fb7da0}, {0xc00102e4e0}, 0xc0a3a98900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc052fb7c80}, 0xc0a3a98900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0543c7050}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805212 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc082889050}, 0xc052c3c300, 0xc048adfc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc082889050}, 0xc052c3c300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041c45121, 0x0, 0x0}, 0xc0973c9280, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041c45121, 0x0, 0x0}, 0xc0973c9280, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc082888fc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082888fc0}, {0xc094f933f8, 0x660089}, 0xc094f93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc082888fc0}, {0x7fe3c2f5f870}, 0xc052c3c100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc082888f30}, 0xc052c3c100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}, 0xc0656eb200, 0xc00c985f38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}, 0x3, 0xc027998f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc094f93980}}}, {0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}, 0xc0656eb200, 0xc027998f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc082888f30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082888f30}, {0x40, 0xc094f93b70}, 0xc094f93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}, 0x203009, 0x203009) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc082888f30}, {0x4e9d7a0, 0xc052c3c100}, 0xc0656eb200, 0xc00c985f38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc082888f30}, 0xc0c8fa7c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05b15fc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05b15fc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809254 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020f4a000, {0x63c85b8, 0xc058fc0540}, 0xc032a1f800, 0xc04e917860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4a000, {0x63c85b8, 0xc058fc0540}, 0xc032a1f800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063a1674b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063a1674b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc058fc04b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058fc04b0}, {0xc0210393f8, 0x660089}, 0xc021039400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc058fc04b0}, {0x3fefffff78658355}, 0xc032a1f500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc058fc0420}, 0xc032a1f500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}, 0xc02fa351c0, 0xc01e9be0c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}, 0x3, 0xc0525cd180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc021039980}}}, {0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}, 0xc02fa351c0, 0xc0525cd180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc058fc0420}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058fc0420}, {0x40, 0xc021039b70}, 0xc021039ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc058fc0420}, {0x4e9d7a0, 0xc032a1f500}, 0xc02fa351c0, 0xc01e9be0c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc058fc0420}, 0xc0228538c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc051eb5b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc051eb5b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804805 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc04404dc80}, 0xc031b68858, 0x1, 0xc0960210a8, 0x519bbc, 0xc06b9a1b60, 0xc03bcf1920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04404dc80}, 0x120, 0xc096021230, 0x6b9a1b00, 0xc01045c9c0, 0x18, 0xc06b9a1b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc04404dc80}, 0xc06b9a1b00, {{{{0x0, 0x0}, {0xc0477b4ea0, 0x19}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04404dc80}, 0x6352410, 0x2f62f5f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc04404dc80}, {0xc06b9a19e0, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc04404dc80}, 0xc05b1132c0, 0xc0960219f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04404dc80}, 0x660089, {0xc06b9a19e0, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc04404dc80}, 0xc0388ff500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04404dbf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04404dbf0}, {0xc0960233f8, 0x660089}, 0xc096023400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04404dbf0}, {0x0}, 0xc0388ff300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04404db60}, 0xc0388ff300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}, 0xc075103280, 0xc01ec01608) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}, 0x3, 0xc06596d840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc096023980}}}, {0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}, 0xc075103280, 0xc06596d840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04404db60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04404db60}, {0x40, 0xc096023b70}, 0xc096023ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}, 0x203019, 0x203019) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04404db60}, {0x4e9d7a0, 0xc0388ff300}, 0xc075103280, 0xc01ec01608) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04404db60}, 0xc0c2845020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af937a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af937a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807002 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bf9fc48, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc077b09d00, 0xc0073d6800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc077b09d00, {0xc0073d6800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc077b09d00, {0xc0073d6800, 0x735, 0xc04e90db80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06618baa0, {0xc0073d6800, 0x7eda79, 0xc0073d6805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc08eca7f80, {0xc0073d6800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04245da10, {0xc0073d6800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc09245c278, {0x631b900, 0xc04245da10}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc09245c000, {0x7fe42ab59310, 0xc08eca7f80}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc09245c000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc09245c000, {0xc05f169000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc05708dab8, {0xc05f169000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc029d90b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc029d90b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc029d910e0, {0x634aac0, 0xc029d90b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc029d90a80, 0xc0924a1230, 0xc00d5cb42b, 0xc00d5cb42d, {0x635bec0, 0xc0187a41b0}, 0xc000d6ae00, 0xc08a4c7d28, 0xc00d5cb429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc029d90a80, {0x63c85b8, 0xc01d6ba990}, 0xc0924a1220, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08751a037, 0x8}}, 0x0, 0xc01d6ba930, 0xc01d6ba960, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc01d6ba990}, {0x643d810, 0xc08eca7f80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01b74be00}, {0x643d810, 0xc08eca7f80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01b74be00}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812624 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc02b87ad80}, 0xc06c871788, 0x1, 0xc043a4b0a8, 0x519bbc, 0xc0a390dce0, 0xc05c3511a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc08edb4960, {0x63c85b8, 0xc02b87ad80}, 0xc02201b680, 0xc043a4b1a8, 0xa390dce0, 0x0, 0x3, 0xc0a390dce0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc02b87ad80}, 0xc0a390dc20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc024f2cd80, 0xb7}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02b87ad80}, 0x6352260, 0x54d3aa98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc02b87ad80}, {0xc0a390d9e0, {0x174301105d4dee7b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02b87ad80}, 0xc066551040, 0xc043a4b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02b87ad80}, 0x660089, {0xc0a390d9e0, {0x174301105d4dee7b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02b87ad80}, 0xc07314fa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105d4dee7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105d4dee7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b87aa50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b87aa50}, {0xc043a4d3f8, 0x660089}, 0xc043a4d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b87aa50}, {0xa00700001005}, 0xc07314f800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02b87a9c0}, 0xc07314f800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}, 0xc019cca140, 0xc060f8e8a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}, 0x3, 0xc015c8e500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043a4d980}}}, {0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}, 0xc019cca140, 0xc015c8e500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02b87a9c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b87a9c0}, {0x40, 0xc043a4db70}, 0xc043a4dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02b87a9c0}, {0x4e9d7a0, 0xc07314f800}, 0xc019cca140, 0xc060f8e8a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02b87a9c0}, 0xc08bd8ba40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356afc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356afc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219640768 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0510a2c60, {0x63c85b8, 0xc0cef7d5f0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc04f01e600, {0x63c85b8, 0xc0cef7d5f0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc04f01e600, {0x63c8510, 0xc0c8eb41c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc04f01e600, {0x63c8510, 0xc0c8eb41c0}, 0xc0118b7ac0, {0x7fe42b790cf0, 0xc0986d46c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc04f01e600, {0x63c85b8, 0xc077420cc0}, {{0x63f88f8, 0xc0118b7ac0}, {0xc012e8ff73, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc04f01e600, {0x63c85b8, 0xc077420cc0}, {{0x63f88f8, 0xc0118b7ac0}, {0xc012e8ff73, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0118b7ac0}, {0xc012e8ff73, 0x23}, 0x0, 0x0}, {0x5fbda7f, 0xedb7a834f, 0x0}, {0x5fbe030, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04f01e600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04f01e600, {0x63c8510, 0xc039c2cfc0}, 0xc07678bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02ef7c930) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc039c2cfc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02ef7c930) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219606922 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af29f78, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc065204080, 0xc0befe4000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc065204080, {0xc0befe4000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc065204080, {0xc0befe4000, 0x735, 0xc0584daa00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03f97e138, {0xc0befe4000, 0x7eda79, 0xc0befe4005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01aa7c240, {0xc0befe4000, 0x62350d4651db0000, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03879f350, {0xc0befe4000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc07d55a978, {0x631b900, 0xc03879f350}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc07d55a700, {0x7fe42ab59310, 0xc01aa7c240}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc07d55a700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc07d55a700, {0xc083b21000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02710d500, {0xc083b21000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06bcfc088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06bcfc088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06bcfc660, {0x634aac0, 0xc06bcfc088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06bcfc000, 0xc08ce8ea60, 0xc067d4f42b, 0xc067d4f42d, {0x635bec0, 0xc04d3c17a0}, 0xc000d6ae00, 0xc0a37410d8, 0xc067d4f429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06bcfc000, {0x63c85b8, 0xc08a924570}, 0xc08ce8ea50, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc07761a009, 0x8}}, 0x0, 0xc08a924510, 0xc08a924540, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc08a924570}, {0x643d810, 0xc01aa7c240}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc048185140}, {0x643d810, 0xc01aa7c240}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc048185140}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801880 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0773c7980}, 0xc07c618800, 0xc01bce37c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0773c7980}, 0xc07c618800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101e0e3064, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101e0e3064, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0773c78c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0773c78c0}, {0xc0608073f8, 0x660089}, 0xc060807400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0773c78c0}, {0xa10000000}, 0xc07c618500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0773c7800}, 0xc07c618500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}, 0xc019f5a640, 0xc078657710) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}, 0x3, 0xc019e9ff80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc060807980}}}, {0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}, 0xc019f5a640, 0xc019e9ff80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0773c7800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0773c7800}, {0x40, 0xc060807b70}, 0xc060807ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0773c7800}, {0x4e9d7a0, 0xc07c618500}, 0xc019f5a640, 0xc078657710) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0773c7800}, 0xc01a4f9500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03c574120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03c574120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805640 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe992d8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc062103700, 0xc08d84c000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc062103700, {0xc08d84c000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc062103700, {0xc08d84c000, 0xd7, 0xc6}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06618a678, {0xc08d84c000, 0xc057717310, 0x1}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01fb9f500, {0xc08d84c000, 0x4a7cec0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02902d9c8, {0xc08d84c000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0c7dc0278, {0x631b900, 0xc02902d9c8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0c7dc0000, {0x7fe42ab59310, 0xc01fb9f500}, 0xc03af10d68) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0c7dc0000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0c7dc0000, {0xc09009b000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0a2f8f620, {0xc09009b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04e84ab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04e84ab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04e84b0e0, {0x634aac0, 0xc04e84ab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04e84aa80, 0xc0790a21a0, 0xc03af1142b, 0xc03af1142d, {0x635bec0, 0xc019c747e0}, 0xc000d6ae00, 0xc03172c198, 0xc03af11429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04e84aa80, {0x63c85b8, 0xc062ca5470}, 0xc0790a2190, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06bd5f04d, 0x8}}, 0x0, 0xc062ca5410, 0xc062ca5440, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc062ca5470}, {0x643d810, 0xc01fb9f500}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc035f0bb90}, {0x643d810, 0xc01fb9f500}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc035f0bb90}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807254 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc044ded440}, 0xc020749bc0, 0x1, 0xc003a930a8, 0x519bbc, 0xc04ff32420, 0xc029abc9c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc044ded440}, 0x1e0, 0xc003a93230, 0x4ff323c0, 0x0, 0x660069, 0xc04ff32420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc044ded440}, 0xc04ff323c0, {{{{0x0, 0x0}, {0xc04bfde240, 0x44}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x10000, {0x63c85b8, 0xc044ded440}, 0x203009, 0x203009) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc044ded440}, {0xc07bb51b00, {0x174301101512e853, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc044ded440}, 0xc07722f2c0, 0xc003a939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044ded440}, 0x660089, {0xc07bb51b00, {0x174301101512e853, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc044ded440}, 0xc08d7d5800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101512e853, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101512e853, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044ded3b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044ded3b0}, {0xc003a953f8, 0x660089}, 0xc003a95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044ded3b0}, {0xc0015221e0}, 0xc08d7d5600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044ded2c0}, 0xc08d7d5600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}, 0xc046e30040, 0xc07349c5b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}, 0x3, 0xc038a3cf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003a95980}}}, {0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}, 0xc046e30040, 0xc038a3cf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044ded2c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044ded2c0}, {0x40, 0xc003a95b70}, 0xc003a95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044ded2c0}, {0x4e9d7a0, 0xc08d7d5600}, 0xc046e30040, 0xc07349c5b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044ded2c0}, 0xc064a1b500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07bb519e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07bb519e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810791 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc035328780, {0x63c85b8, 0xc07995b1a0}, 0xc03a388d20, 0x1, 0xc013b950a8, 0x519bbc, 0xc075b96840, 0xc041f2db60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07995b1a0}, 0x120, 0xc013b95230, 0x75b967e0, 0x0, 0x660069, 0xc075b96840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc035328780, {0x63c85b8, 0xc07995b1a0}, 0xc075b967e0, {{{{0x0, 0x0}, {0xc0639ce000, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07995b1a0}, 0x6352410, 0x55bf2498) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05bb9e870, {0x63c85b8, 0xc07995b1a0}, {0xc075b966c0, {0x174301106e46f97c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05bb9e870, {0x63c85b8, 0xc07995b1a0}, 0xc07076a780, 0xc013b959f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07995b1a0}, 0x660089, {0xc075b966c0, {0x174301106e46f97c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0716f9400, {0x63c85b8, 0xc07995b1a0}, 0xc086e93500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106e46f97c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106e46f97c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07995b110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07995b110}, {0xc013b973f8, 0x660089}, 0xc013b97400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07995b110}, {0xf4}, 0xc086e93300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07995b080}, 0xc086e93300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}, 0xc06734c500, 0xc01c438510) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}, 0x3, 0xc015d27280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc013b97980}}}, {0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}, 0xc06734c500, 0xc015d27280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07995b080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07995b080}, {0x40, 0xc013b97b70}, 0xc013b97ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07995b080}, {0x4e9d7a0, 0xc086e93300}, 0xc06734c500, 0xc01c438510) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07995b080}, 0xc0194c89c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc075b965a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc075b965a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809553 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04f69d0e0}, 0xc01bb1ec18, 0x1, 0xc01fc730a8, 0x519bbc, 0xc08b663e00, 0xc0332dd500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01c50d860, {0x63c85b8, 0xc04f69d0e0}, 0xc033f8ec60, 0xc01fc731a8, 0x8b663e00, 0x0, 0x1, 0xc08b663e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04f69d0e0}, 0xc08b663d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc062ad5d40, 0x80}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f69d0e0}, 0x6352260, 0x46e77b78) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04f69d0e0}, {0xc08b663c20, {0x174301102ad31a60, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04f69d0e0}, 0xc02b2145a0, 0xc01fc739f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f69d0e0}, 0x660089, {0xc08b663c20, {0x174301102ad31a60, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04f69d0e0}, 0xc057e6a700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f69d050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f69d050}, {0xc01fc753f8, 0x660089}, 0xc01fc75400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f69d050}, {0x4d939c}, 0xc057e6a500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f69cfc0}, 0xc057e6a500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}, 0xc04cf67440, 0xc0403c1098) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}, 0x3, 0xc046a145c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fc75980}}}, {0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}, 0xc04cf67440, 0xc046a145c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f69cfc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f69cfc0}, {0x40, 0xc01fc75b70}, 0xc01fc75ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f69cfc0}, {0x4e9d7a0, 0xc057e6a500}, 0xc04cf67440, 0xc0403c1098) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f69cfc0}, 0xc0429f4480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08b663b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08b663b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804652 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc047e9aed0}, 0xc0299e26c0, 0x1, 0x708017d5b0039d, 0x37e11d600, 0xc068294600, 0xc056fde4e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc047e9aed0}, 0x120, 0xc090b61230, 0x682945a0, 0xc00f152340, 0x18, 0xc068294600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc047e9aed0}, 0xc0682945a0, {{{{0x0, 0x0}, {0xc0631c2fc0, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047e9aed0}, 0x6352410, 0x421b1640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc047e9aed0}, {0xc068294480, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc047e9aed0}, 0xc066746320, 0xc090b619f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047e9aed0}, 0x660089, {0xc068294480, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc047e9aed0}, 0xc08dbb3000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e9ae40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e9ae40}, {0xc090b633f8, 0x660089}, 0xc090b63400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e9ae40}, {0x0}, 0xc08dbb2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e9adb0}, 0xc08dbb2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}, 0xc013bea360, 0xc0299e2690) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}, 0x3, 0xc03e23efc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc090b63980}}}, {0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}, 0xc013bea360, 0xc03e23efc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047e9adb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e9adb0}, {0x40, 0xc090b63b70}, 0xc090b63ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}, 0x20300f, 0x20300f) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047e9adb0}, {0x4e9d7a0, 0xc08dbb2e00}, 0xc013bea360, 0xc0299e2690) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047e9adb0}, 0xc01c47ed80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06ab798c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06ab798c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803311 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc0237831d0}, 0xc0c5996c00, 0xc070ea4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0237831d0}, 0xc0c5996c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bfcf779, 0x0, 0x0}, 0xc003a62eb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bfcf779, 0x0, 0x0}, 0xc003a62eb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023782f00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023782f00}, {0xc0762db3f8, 0x660089}, 0xc0762db400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023782f00}, {0x4d939c}, 0xc0c5996a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc023782de0}, 0xc0c5996a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}, 0xc0559188a0, 0xc0689b8810) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}, 0x3, 0xc06b83cd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0762db980}}}, {0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}, 0xc0559188a0, 0xc06b83cd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc023782de0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023782de0}, {0x40, 0xc0762dbb70}, 0xc0762dbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc023782de0}, {0x4e9d7a0, 0xc0c5996a00}, 0xc0559188a0, 0xc0689b8810) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc023782de0}, 0xc027f4e0c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0365c2b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0365c2b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791585 [runnable]: sync.runtime_SemacquireMutex(0xc048727560, 0x0, 0x90) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Delete(0xc00074aa90, 0x37, 0x7, 0x37) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:229 +0x65 github.com/cockroachdb/pebble/internal/cache.(*Cache).Delete(0xc048727560, 0x7fbf, 0x117f, 0xc0a8c7bc80) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:755 +0x3c github.com/cockroachdb/pebble/sstable.(*Writer).writeCompressedBlock(0xc0890ee400, {0xc03c1e2000, 0x7fbf, 0x8000}, {0xc0890eec70, 0xc084744a58, 0x78}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/writer.go:1406 +0x74 github.com/cockroachdb/pebble/sstable.(*writeQueue).performWrite(0xc0286f2a40, 0xc06f8363f0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:59 +0x8b github.com/cockroachdb/pebble/sstable.(*writeQueue).addSync(0xc0286f2a40, 0xc06f8363f0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:119 +0x45 github.com/cockroachdb/pebble/sstable.(*Writer).flush(0xc0890ee400, {{0xc0a8c7bc80, 0xc084744b78, 0x4fce68}, 0x2d}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/writer.go:1091 +0x465 github.com/cockroachdb/pebble/sstable.(*Writer).maybeFlush(0xc0890ee400, {{0xc0a8c7bc80, 0x792c35ef0c, 0xc0a8c7bc80}, 0x11e3fc6}, {0xc0aead2000, 0xc03b9db078, 0x1}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/writer.go:1101 +0x87 github.com/cockroachdb/pebble/sstable.(*Writer).addPoint(0xc0890ee400, {{0xc0a8c7bc80, 0x7fe42ab8b8d0, 0x1}, 0x2}, {0xc0aead2000, 0x5, 0x8e000}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/writer.go:624 +0x353 github.com/cockroachdb/pebble/sstable.(*Writer).Add(0xc003134c60, {{0xc0a8c7bc80, 0x5a, 0xc0a8c7bc80}, 0x91a9ee0}, {0xc0aead2000, 0x63687f0, 0x51c2e30}) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/writer.go:588 +0xa9 github.com/cockroachdb/pebble.(*DB).runCompaction(0xc0002a1680, 0xf63c, 0xc06ae18840, {0x63247e0, 0x9bf6420}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:2390 +0x1e6d github.com/cockroachdb/pebble.(*DB).compact1(0xc0002a1680, 0xc06ae18840, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1889 +0x2de github.com/cockroachdb/pebble.(*DB).compact.func1({0x63c85b8, 0xc01f305860}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1850 +0xad runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc0001670e0, 0x6446ff8, 0xc019573860}}, 0xc031b37f88) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble.(*DB).compact(0xad08a0, 0xc0131fd400, 0xc031b37fb8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1847 +0x6b created by github.com/cockroachdb/pebble.(*DB).maybeScheduleCompactionPicker github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/compaction.go:1641 +0x5c5 goroutine 219809185 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc018b46300}, 0xc0824c5600, 0xc05953ee60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc018b46300}, 0xc0824c5600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066f2ec44, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066f2ec44, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077e8df80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077e8df80}, {0xc0127f53f8, 0x660089}, 0xc0127f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077e8df80}, {0x0}, 0xc0824c5300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077e8def0}, 0xc0824c5300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}, 0xc06db6fc40, 0xc02278b518) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}, 0x3, 0xc058930440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0127f5980}}}, {0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}, 0xc06db6fc40, 0xc058930440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc077e8def0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077e8def0}, {0x40, 0xc0127f5b70}, 0xc0127f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc077e8def0}, {0x4e9d7a0, 0xc0824c5300}, 0xc06db6fc40, 0xc02278b518) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc077e8def0}, 0xc07a063860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0877a06c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0877a06c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803918 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc011b911d0}, 0xc057eb0438, 0x1, 0xc078e6d0a8, 0x519bbc, 0xc05859ea80, 0xc051b56cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc011b911d0}, 0x120, 0xc078e6d230, 0x5859ea20, 0x0, 0x660069, 0xc05859ea80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc011b911d0}, 0xc05859ea20, {{{{0x0, 0x0}, {0xc04d732240, 0xef}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc011b911d0}, 0x6352410, 0x421b0a88) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc011b911d0}, {0xc05859e900, {0x17430110321c6705, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc011b911d0}, 0xc057b34780, 0xc078e6d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc011b911d0}, 0x660089, {0xc05859e900, {0x17430110321c6705, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc011b911d0}, 0xc05f621d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110321c6705, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110321c6705, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc011b91140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc011b91140}, {0xc078e6f3f8, 0x660089}, 0xc078e6f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc011b91140}, {0xc0015221e0}, 0xc05f621900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc011b910b0}, 0xc05f621900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}, 0xc05393e940, 0xc07a3e0cf0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}, 0x3, 0xc060a5d7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078e6f980}}}, {0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}, 0xc05393e940, 0xc060a5d7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc011b910b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc011b910b0}, {0x40, 0xc078e6fb70}, 0xc078e6fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc011b910b0}, {0x4e9d7a0, 0xc05f621900}, 0xc05393e940, 0xc07a3e0cf0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc011b910b0}, 0xc07af8db60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05859e7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05859e7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801016 [runnable]: sync.runtime_SemacquireMutex(0xc09696c690, 0x8f, 0x7fe1b34b7f40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc09696c538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc09696c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0b9185b00, {0xffffffffffffff15, 0xc09696c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0b9185b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0b9185b00, {0xc002bb5c70, 0x32, 0x70}, {0xc015d7ab00, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0923bed98, {0xc002bb5c70, 0x32, 0x70}, {0xc015d7ab00, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0923bd6c0, {0xc015d7ab00, 0x77, 0x70}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0923bd6c0, {0xc002bb5c70, 0x77, 0xc015d7ab00}, {0xc015d7ab00, 0xc0689f4e10, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0923bd400, {0xc015d7ab00, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc064e52828, {{0xc0689f4e10, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06f9e5700, {{0xc04fcb3050, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc062662000, {0x63c85b8, 0xc03988dfb0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03988dfb0}, {0x64a6560, 0xc06f9e5700}, {0xc04fcb3050, 0xc08307da40, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03988dfb0}, {0x6466ed8, 0xc064e52580}, {0xc04fcb3050, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04318ff98}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04b06d900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c20c00, {0x63c85b8, 0xc03988dfb0}, 0xc04b06d900, 0xc055fb8b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c20c00, {0x63c85b8, 0xc03988dfb0}, 0xc04b06d900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03988df20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03988df20}, {0xc07d2528f0, 0x660089}, 0xc07d2528f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03988df20}, {0xc00102e4e0}, 0xc04b06d800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03988dec0}, 0xc04b06d800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a4d20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219150574 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc0a1948380}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc076c05220}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}, 0xc045e68f30, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}, 0x3, 0xc01dcb0500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x16f0987}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}, 0xc045e68f30, 0xc01dcb0500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f4bec30}, {0x460ef4, 0xc007b5abc8}, 0xc007b5ab70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}, 0xc045e68f30, 0xc01dcb0500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc007d30680}, 0xc045e68f30, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc0b00d4d00}, 0xc06521d8c0, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0b00d4d00}, 0xc06521d8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800631 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea35e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07d061600, 0xc02916a800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07d061600, {0xc02916a800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07d061600, {0xc02916a800, 0x735, 0xc066bd9680}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03fdaab80, {0xc02916a800, 0x7eda79, 0xc02916a805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0330803c0, {0xc02916a800, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02f753878, {0xc02916a800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0465b4978, {0x631b900, 0xc02f753878}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0465b4700, {0x7fe42ab59310, 0xc0330803c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0465b4700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0465b4700, {0xc020b21000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc08526b728, {0xc020b21000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0b6dca088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0b6dca088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0b6dca660, {0x634aac0, 0xc0b6dca088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0b6dca000, 0xc046054380, 0xc08847942b, 0xc08847942d, {0x635bec0, 0xc024c55ef0}, 0xc000d6ae00, 0xc03c759d28, 0xc088479429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0b6dca000, {0x63c85b8, 0xc06b9b33e0}, 0xc046054370, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0335e2047, 0x8}}, 0x0, 0xc06b9b3380, 0xc06b9b33b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06b9b33e0}, {0x643d810, 0xc0330803c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0613d66c0}, {0x643d810, 0xc0330803c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0613d66c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803827 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021daa000, {0x63c85b8, 0xc06e12ec00}, 0xc04835fb00, 0xc04a8cf9a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021daa000, {0x63c85b8, 0xc06e12ec00}, 0xc04835fb00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032116a94, 0x0, 0x0}, 0xc01d10f130, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032116a94, 0x0, 0x0}, 0xc01d10f130, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e12eb70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e12eb70}, {0xc0624453f8, 0x660089}, 0xc062445400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e12eb70}, {0x0}, 0xc04835f900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e12eae0}, 0xc04835f900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}, 0xc01dc38d40, 0xc03d3e8180) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}, 0x3, 0xc0aa751100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062445980}}}, {0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}, 0xc01dc38d40, 0xc0aa751100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06e12eae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e12eae0}, {0x40, 0xc062445b70}, 0xc062445ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06e12eae0}, {0x4e9d7a0, 0xc04835f900}, 0xc01dc38d40, 0xc03d3e8180) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06e12eae0}, 0xc053f86120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04e55fe60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04e55fe60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805647 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc016ba0000, {0x63c85b8, 0xc02b26ba70}, 0xc063449900, 0xc048083ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016ba0000, {0x63c85b8, 0xc02b26ba70}, 0xc063449900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b26b770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b26b770}, {0xc005ae73f8, 0x660089}, 0xc005ae7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b26b770}, {0xc0015221e0}, 0xc063449700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02b26b6b0}, 0xc063449700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}, 0xc06deb8ee0, 0xc07d345590) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}, 0x3, 0xc0639a3d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ae7980}}}, {0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}, 0xc06deb8ee0, 0xc0639a3d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02b26b6b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b26b6b0}, {0x40, 0xc005ae7b70}, 0xc005ae7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02b26b6b0}, {0x4e9d7a0, 0xc063449700}, 0xc06deb8ee0, 0xc07d345590) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02b26b6b0}, 0xc02ecadf20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc08bd859e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc08bd859e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804963 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc04ef99980}, 0xc0299e3cc8, 0x1, 0x708017d839988e, 0x37e11d600, 0xc06eaf24e0, 0xc06f6fe060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04ef99980}, 0x120, 0xc0889cd230, 0x6eaf2480, 0xc00f152ea0, 0x18, 0xc06eaf24e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc04ef99980}, 0xc06eaf2480, {{{{0x0, 0x0}, {0xc0604926c0, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ef99980}, 0x6352410, 0x421b1858) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc04ef99980}, {0xc06eaf2360, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc04ef99980}, 0xc022382140, 0xc0889cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ef99980}, 0x660089, {0xc06eaf2360, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc04ef99980}, 0xc01aa10d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ef998f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef998f0}, {0xc0889cf3f8, 0x660089}, 0xc0889cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ef998f0}, {0xc0225653f8}, 0xc01aa10900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ef99860}, 0xc01aa10900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}, 0xc013beb8a0, 0xc0299e3c98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}, 0x3, 0xc06f13cec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0889cf980}}}, {0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}, 0xc013beb8a0, 0xc06f13cec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ef99860}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef99860}, {0x40, 0xc0889cfb70}, 0xc0889cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ef99860}, {0x4e9d7a0, 0xc01aa10900}, 0xc013beb8a0, 0xc0299e3c98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ef99860}, 0xc0415357a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06f5abc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06f5abc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801189 [runnable]: sync.runtime_SemacquireMutex(0xc0961c8690, 0x8f, 0xc0961c86a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0961c8538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0961c8560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc07472c000, {0xffffffffffffff15, 0xc0961c86c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc07472c000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc07472c000, {0xc0db314bd0, 0x32, 0x6d}, {0xc016b18c00, 0x32, 0x78}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07a95cd98, {0xc0db314bd0, 0x32, 0x6d}, {0xc016b18c00, 0x32, 0x78}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07a95b6c0, {0xc016b18c00, 0x78, 0x6d}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07a95b6c0, {0xc0db314bd0, 0x78, 0xc016b18c00}, {0xc016b18c00, 0xc07005f1d0, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07a95b400, {0xc016b18c00, 0x32, 0x78}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc05429e2a8, {{0xc07005f1d0, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc067ff4300, {{0xc0654a5020, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc059f3d000, {0x63c85b8, 0xc0487b21e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0487b21e0}, {0x64a6560, 0xc067ff4300}, {0xc0654a5020, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0487b21e0}, {0x6466ed8, 0xc05429e000}, {0xc0654a5020, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04318f560}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc058269800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c24800, {0x63c85b8, 0xc0487b21e0}, 0xc058269800, 0xc022f4a780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c24800, {0x63c85b8, 0xc0487b21e0}, 0xc058269800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0487b2120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0487b2120}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0487b2120}, {0xc00102e4e0}, 0xc058269700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0487b20c0}, 0xc058269700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc076fafa40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805409 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0310c8140, {0x63c85b8, 0xc029d64150}, 0xc0525c1bc0, 0x1, 0xc07b0f30a8, 0x519bbc, 0xc05fcb6600, 0xc05f6736e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc029d64150}, 0x120, 0xc07b0f3230, 0x5fcb65a0, 0xc00f153040, 0x18, 0xc05fcb6600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0310c8140, {0x63c85b8, 0xc029d64150}, 0xc05fcb65a0, {{{{0x0, 0x0}, {0xc0570a4360, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc029d64150}, 0x6352410, 0x3a07e530) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc03107e870, {0x63c85b8, 0xc029d64150}, {0xc05fcb6480, {0x1743011044d2d939, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc03107e870, {0x63c85b8, 0xc029d64150}, 0xc05292efa0, 0xc07b0f39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc029d64150}, 0x660089, {0xc05fcb6480, {0x1743011044d2d939, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310be400, {0x63c85b8, 0xc029d64150}, 0xc09f3dab00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011044d2d939, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011044d2d939, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029d640c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029d640c0}, {0xc07b0f53f8, 0x660089}, 0xc07b0f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029d640c0}, {0x7fe423ece940}, 0xc09f3da900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029d64030}, 0xc09f3da900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}, 0xc021f06fe0, 0xc0234eb200) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}, 0x3, 0xc017dedd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b0f5980}}}, {0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}, 0xc021f06fe0, 0xc017dedd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029d64030}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029d64030}, {0x40, 0xc07b0f5b70}, 0xc07b0f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}, 0x203005, 0x203005) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029d64030}, {0x4e9d7a0, 0xc09f3da900}, 0xc021f06fe0, 0xc0234eb200) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029d64030}, 0xc0431c71a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05fcb6360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05fcb6360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811891 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0b2c00, {0x63c85b8, 0xc023b27ef0}, 0xc02029d700, 0xc02faf6e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc023b27ef0}, 0xc02029d700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b1b7318, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b1b7318, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023b27e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023b27e60}, {0xc034e893f8, 0x660089}, 0xc034e89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023b27e60}, {0x7fe34af0d658}, 0xc02029d500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc023b27da0}, 0xc02029d500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}, 0xc08d07aae0, 0xc021240570) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}, 0x3, 0xc01586df80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034e89980}}}, {0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}, 0xc08d07aae0, 0xc01586df80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc023b27da0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023b27da0}, {0x40, 0xc034e89b70}, 0xc034e89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc023b27da0}, {0x4e9d7a0, 0xc02029d500}, 0xc08d07aae0, 0xc021240570) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc023b27da0}, 0xc065ea5980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02b507d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02b507d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803513 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc00d364d20}, 0xc07a5c2930, 0x1, 0xc0764bd0a8, 0x519bbc, 0xc03c402720, 0xc03a90bc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc00d364d20}, 0x120, 0xc0764bd230, 0x3c4026c0, 0x0, 0x660069, 0xc03c402720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc00d364d20}, 0xc03c4026c0, {{{{0x0, 0x0}, {0xc048a1bb00, 0xe0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc00d364d20}, 0x6352410, 0x45bf43a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc00d364d20}, {0xc03c4025a0, {0x174301102bca3600, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc00d364d20}, 0xc0742b1680, 0xc0764bd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc00d364d20}, 0x660089, {0xc03c4025a0, {0x174301102bca3600, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc00d364d20}, 0xc038cec400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bca3600, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bca3600, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc00d364c30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00d364c30}, {0xc0764bf3f8, 0x660089}, 0xc0764bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc00d364c30}, {0x0}, 0xc038cec200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc00d364a80}, 0xc038cec200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}, 0xc0350508c0, 0xc0379d1788) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}, 0x3, 0xc020de74c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0764bf980}}}, {0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}, 0xc0350508c0, 0xc020de74c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc00d364a80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00d364a80}, {0x40, 0xc0764bfb70}, 0xc0764bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc00d364a80}, {0x4e9d7a0, 0xc038cec200}, 0xc0350508c0, 0xc0379d1788) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc00d364a80}, 0xc022266360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03c402480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03c402480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219150417 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0462a2630, {0x63c85b8, 0xc034f9e270}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0a3c09300, {0x63c85b8, 0xc034f9e270}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0a3c09300, {0x63c8510, 0xc03a6c6980}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc0a3c09300, {0x63c8510, 0xc03a6c6980}, {0x63f7db8, 0x9bf6f48}, 0xc081d7b088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0a3c09300, {0x63c85b8, 0xc02c33ef90}, {{0x63f7db8, 0x9bf6f48}, {0xc0667ddc6a, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0a3c09300, {0x63c85b8, 0xc02c33ef90}, {{0x63f7db8, 0x9bf6f48}, {0xc0667ddc6a, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0667ddc6a, 0x6}, 0x0, 0x0}, {0x114c34eb, 0xedb7a8350, 0x0}, {0x114c3931, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0a3c09300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0a3c09300, {0x63c8510, 0xc05bdc0240}, 0xc071fa9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02856c890) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc05bdc0240}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02856c890) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812802 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804506 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc054c66330}, 0xc0a4f84300, 0xc022ff1540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc054c66330}, 0xc0a4f84300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011033dddbad, 0x0, 0x0}, 0xc066e96590, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011033dddbad, 0x0, 0x0}, 0xc066e96590, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc054c66240}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054c66240}, {0xc0790733f8, 0x660089}, 0xc079073400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc054c66240}, {0x0}, 0xc0a4f84100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc054c661b0}, 0xc0a4f84100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}, 0xc076403b80, 0xc028550498) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}, 0x3, 0xc0510795c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079073980}}}, {0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}, 0xc076403b80, 0xc0510795c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc054c661b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054c661b0}, {0x40, 0xc079073b70}, 0xc079073ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc054c661b0}, {0x4e9d7a0, 0xc0a4f84100}, 0xc076403b80, 0xc028550498) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc054c661b0}, 0xc030f64d20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f138000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f138000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 209252103 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc01776ab10, 0x2c5f) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc078274000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc078274660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc059a72600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc059a72600, {0x63c8510, 0xc01776ab40}, 0xc04a165dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03f4243a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc01776ab40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03f4243a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219805721 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0309e0800, {0x63c85b8, 0xc0496edf20}, 0xc05f10b300, 0xc04c1cf720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0309e0800, {0x63c85b8, 0xc0496edf20}, 0xc05f10b300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104bbdda8b, 0x0, 0x0}, 0xc076d72ed0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104bbdda8b, 0x0, 0x0}, 0xc076d72ed0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0496ede90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0496ede90}, {0xc06f205918, 0x660089}, 0xc06f205920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0496ede90}, {0xc00102e4e0}, 0xc05f10b200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0496ede30}, 0xc05f10b200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc04e9cdec0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0496edda0}, {0xc00dfd36b0, 0x24}, 0x46c7b7781, 0xc015bc2d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x10, 0xc0199d0478}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0496edda0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812319 [select]: github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:72 +0xe8 created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:67 +0x65 goroutine 219807004 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc05035cdd0, {0x63c85b8, 0xc0436553b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0436553b0}, {0x4f67ad2, 0x6446ff8}, 0xc04da06fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219802570 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc09996e548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc09996e5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc092898cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0539aca80, {0xc01e28b800, 0x2b, 0x74}, {0xc01bfabd00, 0x2b, 0x36}, 0x1, 0x8c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092898b88, {0xc01e28b800, 0x1, 0x36}, {0xc01bfabd00, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092898b88, {0xc01e28b800, 0x2b, 0x74}, {0xc01bfabd00, 0x2b, 0x36}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0928982c0, {0xc01bfabd00, 0x36, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0928982c0, {0xc01e28b800, 0x2b, 0xc01bfabd00}, {0xc01bfabd00, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092898000, {0xc01bfabd00, 0x2b, 0x36}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04300e688, {{0xc054f745a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04c765700, {{0xc054f745a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f958400, {0x63c85b8, 0xc041c9b080}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc041c9b080}, {0x64a6560, 0xc04c765700}, {0xc054f745a0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc041c9b080}, {0x6466ed8, 0xc04300e680}, {0xc054f745a0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02e7cf2c0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc062531800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f1f000, {0x63c85b8, 0xc041c9b080}, 0xc062531800, 0xc04acddc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f1f000, {0x63c85b8, 0xc041c9b080}, 0xc062531800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041c9aff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041c9aff0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041c9aff0}, {0xc00102e4e0}, 0xc062531700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041c9af90}, 0xc062531700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220990}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812360 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0606698d0, {0x63c85b8, 0xc07d21f2f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc07d21f2f0}, {0x4f67ad2, 0x10beec0}, 0xc002c9efb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219809169 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021a5f7c0, {0x63c85b8, 0xc02ecf5b60}, 0xc01365b170, 0x1, 0xc0072070a8, 0x519bbc, 0xc0c37dc060, 0xc0610ce720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02ecf5b60}, 0x120, 0xc007207230, 0xc37dc000, 0x0, 0x660069, 0xc0c37dc060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021a5f7c0, {0x63c85b8, 0xc02ecf5b60}, 0xc0c37dc000, {{{{0x0, 0x0}, {0xc0873f99e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02ecf5b60}, 0x6352410, 0x4b1bc470) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a30fa0, {0x63c85b8, 0xc02ecf5b60}, {0xc053469e60, {0x17430110638c955a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a30fa0, {0x63c85b8, 0xc02ecf5b60}, 0xc083df41e0, 0xc0072079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02ecf5b60}, 0x660089, {0xc053469e60, {0x17430110638c955a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021ab8000, {0x63c85b8, 0xc02ecf5b60}, 0xc05ab62800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110638c955a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110638c955a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ecf5ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ecf5ad0}, {0xc0072093f8, 0x660089}, 0xc007209400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ecf5ad0}, {0x4071bff3e68e5eba}, 0xc05ab62600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ecf5a40}, 0xc05ab62600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}, 0xc0ce315f00, 0xc02133bf08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}, 0x3, 0xc02dd3a0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007209980}}}, {0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}, 0xc0ce315f00, 0xc02dd3a0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02ecf5a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ecf5a40}, {0x40, 0xc007209b70}, 0xc007209ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02ecf5a40}, {0x4e9d7a0, 0xc05ab62600}, 0xc0ce315f00, 0xc02133bf08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02ecf5a40}, 0xc054ae7200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc053469d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc053469d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794628 [runnable]: sync.runtime_SemacquireMutex(0xc08d9ce690, 0x8f, 0x7fe3f7e3c900) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc08d9ce538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc08d9ce560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0912a2d80, {0xffffffffffffff15, 0xc08d9ce6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0912a2d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0912a2d80, {0xc005118620, 0x32, 0x63}, {0xc0da8a8200, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc096dc2898, {0xc005118620, 0x32, 0x63}, {0xc0da8a8200, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc096dc11c0, {0xc0da8a8200, 0x74, 0x63}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc096dc11c0, {0xc005118620, 0x74, 0xc0da8a8200}, {0xc0da8a8200, 0xc052fab2c0, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc096dc0f00, {0xc0da8a8200, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06c035e28, {{0xc052fab2c0, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc00436ec00, {{0xc04464a1b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0833d2000, {0x63c85b8, 0xc081eb2810}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc081eb2810}, {0x64a6560, 0xc00436ec00}, {0xc04464a1b0, 0xc011912600, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc081eb2810}, {0x6466ed8, 0xc06c035b80}, {0xc04464a1b0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04318f1e8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04ba6e100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020079000, {0x63c85b8, 0xc081eb2810}, 0xc04ba6e100, 0xc05ffce460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020079000, {0x63c85b8, 0xc081eb2810}, 0xc04ba6e100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc081eb25a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc081eb25a0}, {0xc09365a8f0, 0x660089}, 0xc09365a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc081eb25a0}, {0x24}, 0xc04ba6e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc081eb24e0}, 0xc04ba6e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03db99f20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810096 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0327d2e10}, 0xc0668d09f0, 0x1, 0xc005d6f0a8, 0x519bbc, 0xc043aa4420, 0xc0a834f140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0327d2e10}, 0x1e0, 0xc005d6f230, 0x43aa43c0, 0x0, 0x660069, 0xc043aa4420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0327d2e10}, 0xc043aa43c0, {{{{0x0, 0x0}, {0xc078937680, 0x5d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0327d2e10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0327d2e10}, {0xc062fb6fc0, {0x174301106b8205d4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0327d2e10}, 0xc02405ab40, 0xc005d6f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0327d2e10}, 0x660089, {0xc062fb6fc0, {0x174301106b8205d4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0327d2e10}, 0xc041cc4400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106b8205d4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106b8205d4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0327d2d80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0327d2d80}, {0xc005d713f8, 0x660089}, 0xc005d71400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0327d2d80}, {0xc06f053f80}, 0xc041cc4200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0327d2cf0}, 0xc041cc4200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}, 0xc06fc909e0, 0xc06ea01d58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}, 0x3, 0xc06cc44180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005d71980}}}, {0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}, 0xc06fc909e0, 0xc06cc44180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0327d2cf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0327d2cf0}, {0x40, 0xc005d71b70}, 0xc005d71ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0327d2cf0}, {0x4e9d7a0, 0xc041cc4200}, 0xc06fc909e0, 0xc06ea01d58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0327d2cf0}, 0xc03d0ed200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07a802480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07a802480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803868 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc031194500, {0x63c85b8, 0xc06be75b30}, 0xc0435fb800, 0x1, 0xc0685a50a8, 0x519bbc, 0xc04cfe2720, 0xc03c14f140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06be75b30}, 0x120, 0xc0685a5230, 0x4cfe26c0, 0x0, 0x660069, 0xc04cfe2720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc031194500, {0x63c85b8, 0xc06be75b30}, 0xc04cfe26c0, {{{{0x0, 0x0}, {0xc0c8404360, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06be75b30}, 0x6352410, 0x34330ad0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc03107f9f0, {0x63c85b8, 0xc06be75b30}, {0xc04cfe25a0, {0x174301102d39292f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc03107f9f0, {0x63c85b8, 0xc06be75b30}, 0xc030698be0, 0xc0685a59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06be75b30}, 0x660089, {0xc04cfe25a0, {0x174301102d39292f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031189400, {0x63c85b8, 0xc06be75b30}, 0xc04c3f9700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d39292f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d39292f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06be75aa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06be75aa0}, {0xc0685a73f8, 0x660089}, 0xc0685a7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06be75aa0}, {0x17a1c0a1fae8a8f2}, 0xc04c3f9400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06be75a10}, 0xc04c3f9400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}, 0xc061a13620, 0xc012a48f30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}, 0x3, 0xc015b69d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0685a7980}}}, {0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}, 0xc061a13620, 0xc015b69d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06be75a10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06be75a10}, {0x40, 0xc0685a7b70}, 0xc0685a7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06be75a10}, {0x4e9d7a0, 0xc04c3f9400}, 0xc061a13620, 0xc012a48f30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06be75a10}, 0xc0228f3740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04cfe2480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04cfe2480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812519 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810009 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04b6d0a50}, 0xc038bf2948, 0x1, 0xc01a7b50a8, 0x519bbc, 0xc04efebf00, 0xc091a5a420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc001760060, {0x63c85b8, 0xc04b6d0a50}, 0xc03fc3a480, 0xc01a7b51a8, 0x1c97ccf, 0xc068122a20, 0x2, 0xc04efebf00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04b6d0a50}, 0xc021d34240, {{{{0xc079db7b00, 0x46}, {0xc019560fc0, 0x88}}, {{0xc028414b40, 0x9f}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04b6d0a50}, 0x63521a0, 0x1fa2f860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04b6d0a50}, {0xc03fc71560, {0x1743011011531b97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04b6d0a50}, 0xc050b1afa0, 0xc01a7b59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04b6d0a50}, 0x660089, {0xc03fc71560, {0x1743011011531b97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04b6d0a50}, 0xc057b95000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011011531b97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011011531b97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04b6d09c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04b6d09c0}, {0xc01a7b73f8, 0x660089}, 0xc01a7b7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04b6d09c0}, {0x3}, 0xc057b94d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04b6d0900}, 0xc057b94d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}, 0xc08dc237c0, 0xc05b127d10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}, 0x3, 0xc018243280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a7b7980}}}, {0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}, 0xc08dc237c0, 0xc018243280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04b6d0900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04b6d0900}, {0x40, 0xc01a7b7b70}, 0xc01a7b7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04b6d0900}, {0x4e9d7a0, 0xc057b94d00}, 0xc08dc237c0, 0xc05b127d10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04b6d0900}, 0xc021d34060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03fc71320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03fc71320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804519 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc034ee7830}, 0xc052163b78, 0x1, 0x0, 0x1, 0xc05f05a960, 0xc05ebad6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc034ee7830}, 0x120, 0xc078ca2728, 0x5f05a900, 0x0, 0x660069, 0xc05f05a960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc034ee7830}, 0xc05f05a900, {{{{0x0, 0x0}, {0xc073da6240, 0xd}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc034ee7830}, 0x6352410, 0x421b0c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc034ee7830}, {0xc05f05a6c0, {0x17430110323939d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc034ee7830}, 0xc08745a960, 0xc057c74ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc034ee7830}, 0x660089, {0xc05f05a6c0, {0x17430110323939d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc034ee7830}, 0xc046c98e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110323939d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110323939d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc034ee77a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc034ee77a0}, {0xc0936928f0, 0x660089}, 0xc0936928f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc034ee77a0}, {0xc00102e4e0}, 0xc046c98d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc034ee7740}, 0xc046c98d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc034ee73b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219492906 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc097747600, {0x63c85b8, 0xc083279a40}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc032835900, {0x63c85b8, 0xc083279a40}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc032835900, {0x63c8510, 0xc07c15a700}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc032835900, {0x63c8510, 0xc07c15a700}, {0x63f7db8, 0x9bf6f48}, 0xc07d0b5088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc032835900, {0x63c85b8, 0xc088057140}, {{0x63f7db8, 0x9bf6f48}, {0xc014851310, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc032835900, {0x63c85b8, 0xc088057140}, {{0x63f7db8, 0x9bf6f48}, {0xc014851310, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc014851310, 0x6}, 0x0, 0x0}, {0x33a8f29d, 0xedb7a834f, 0x0}, {0x33a8f948, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc032835900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc032835900, {0x63c8510, 0xc0665b5180}, 0xc05730ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03da60360) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0665b5180}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03da60360) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808602 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc05ca98dc0, {0x63c85b8, 0xc04194b620}, 0xc02c5ece88, 0x1, 0xc0071370a8, 0x519bbc, 0xc064c9f2c0, 0xc041b789c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc042ec57a0, {0x63c85b8, 0xc04194b620}, 0xc042ec57e8, 0xc0071371a8, 0x50, 0x70, 0xffffffffffffffff, 0xc064c9f2c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc05ca98dc0, {0x63c85b8, 0xc04194b620}, 0xc064c9f200, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc074adb7a0, 0xc}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04194b620}, 0x6352260, 0x570841c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc045008410, {0x63c85b8, 0xc04194b620}, {0xc064c9f0e0, {0x1743011020584514, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc045008410, {0x63c85b8, 0xc04194b620}, 0xc068588280, 0xc0071379f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04194b620}, 0x660089, {0xc064c9f0e0, {0x1743011020584514, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc06ffeec00, {0x63c85b8, 0xc04194b620}, 0xc05ad1c000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020584514, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020584514, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04194b560}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04194b560}, {0xc0071393f8, 0x660089}, 0xc007139400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04194b560}, {0x0}, 0xc04682fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04194b4d0}, 0xc04682fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}, 0xc0993a9e20, 0xc015c03110) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}, 0x3, 0xc0374e61c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007139980}}}, {0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}, 0xc0993a9e20, 0xc0374e61c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04194b4d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04194b4d0}, {0x40, 0xc007139b70}, 0xc007139ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04194b4d0}, {0x4e9d7a0, 0xc04682fe00}, 0xc0993a9e20, 0xc015c03110) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04194b4d0}, 0xc0c90f8f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc064c9efc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc064c9efc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812577 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812780 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810350 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035040, {0x63c85b8, 0xc021131260}, 0xc02a7aba70, 0x1, 0xc07bc43920, 0x519bbc, 0xc0d19a9020, 0xc053ff5140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05abc19e0, {0x63c85b8, 0xc021131260}, 0xc05abc1a18, 0xc07bc43a20, 0x41, 0x60, 0xffffffffffffffff, 0xc0d19a9020, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035040, {0x63c85b8, 0xc021131260}, 0xc0d19a8fc0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc077e2f680, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc021131260}, 0x63523b0, 0x7818fc18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd3b0, {0x63c85b8, 0xc021131260}, {0x0, {0x1743011074fe43c8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd3b0, {0x63c85b8, 0xc021131260}, 0xc0777e3c20, 0xc07bc44268) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc021131260}, 0x660089, {0x0, {0x1743011074fe43c8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc021131260}, 0xc08ba2be00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021131170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021131170}, {0xc00dd97c70, 0x660089}, 0xc00dd97c78) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021131170}, {0xc03b81ed50}, 0xc08ba2bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021130f30}, 0xc08ba2bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x1743011074fe43c8, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc03b81eff0}, 0x0, 0xc089d62c00) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc03b81eff0}, 0x19) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0078dea50, {0x63c85b8, 0xc03b81eff0}, 0xc00dd9a6b0, {{0x1743011074fe2ebe, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:392 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0x0, {0x63c85b8, 0xc03b81eff0}, 0xc04474e398, {{0x1743011074fe2ebe, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x165 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc02dfdd400, {0x63c85b8, 0xc03b81eff0}, {0xc070e40120, {0x174301106dc7bef5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:499 +0x533 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc03b81eff0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc02dfdd400, {0x63c85b8, 0xc03b81eff0}, {0xc070e40120, {0x174301106dc7bef5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd3b0, {0x63c85b8, 0xc03b81eff0}, 0xc086299b80, 0xc00dd9b558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03b81eff0}, 0xc086299b80, {0xc070e40120, {0x174301106dc7bef5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc03b81eff0}, 0xc058ec8e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106dc7bef5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106dc7bef5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03b81ef30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b81ef30}, {0xc00dd9d3f8, 0x660089}, 0xc00dd9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03b81ef30}, {0x3fefffff57ae8673}, 0xc058ec8b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03b81eea0}, 0xc058ec8b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}, 0xc00c7ee000, 0xc063fa79b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}, 0x3, 0xc06cbd4900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00dd9d980}}}, {0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}, 0xc00c7ee000, 0xc06cbd4900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03b81eea0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b81eea0}, {0x40, 0xc00dd9db08}, 0xc00dd9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03b81eea0}, {0x4e9d7a0, 0xc058ec8b00}, 0xc00c7ee000, 0xc063fa79b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03b81eea0}, 0xc0bd799bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01049be60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01049be60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811898 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc01acfe7e0}, 0xc063faf9c8, 0x1, 0x7080181a22cc00, 0x37e11d600, 0xc02895d500, 0xc0516d0420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc01acfe7e0}, 0x1e0, 0xc01f2f7230, 0x2895d4a0, 0x0, 0x660069, 0xc02895d500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc01acfe7e0}, 0xc02895d4a0, {{{{0x0, 0x0}, {0xc0a626ed80, 0x6c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01acfe7e0}, 0x63521a0, 0x3c90c110) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc01acfe7e0}, {0xc04cabb9e0, {0x174301107b1cf270, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc01acfe7e0}, 0xc0631c5860, 0xc01f2f79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01acfe7e0}, 0x660089, {0xc04cabb9e0, {0x174301107b1cf270, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc01acfe7e0}, 0xc0d29ed400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b1cf270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b1cf270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01acfe720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01acfe720}, {0xc01f2f93f8, 0x660089}, 0xc01f2f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01acfe720}, {0x7fdd4a3e6eb8}, 0xc0d29ed200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01acfe690}, 0xc0d29ed200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}, 0xc04c986100, 0xc063faf998) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}, 0x3, 0xc05426d780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f2f9980}}}, {0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}, 0xc04c986100, 0xc05426d780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01acfe690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01acfe690}, {0x40, 0xc01f2f9b70}, 0xc01f2f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01acfe690}, {0x4e9d7a0, 0xc0d29ed200}, 0xc04c986100, 0xc063faf998) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01acfe690}, 0xc02f146f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04cabb8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04cabb8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218980147 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3df500, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07273c580, 0xc097b3b000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07273c580, {0xc097b3b000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07273c580, {0xc097b3b000, 0x735, 0xc0bb64b680}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0420123c8, {0xc097b3b000, 0x100, 0xff}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01bb358c0, {0xc097b3b000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a90408, {0xc097b3b000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc062d07e78, {0x631b900, 0xc043a90408}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc062d07c00, {0x7fe42ab59310, 0xc01bb358c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc062d07c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc062d07c00, {0xc020b25000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc071bdb770, {0xc020b25000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc020b19588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc020b19588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc020b19b60, {0x634aac0, 0xc020b19588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc020b19500, 0xc04727e290, 0xc0bbced42b, 0xc0bbced42d, {0x635bec0, 0xc011605320}, 0xc000d6ae00, 0xc044cce838, 0xc0bbced429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc020b19500, {0x63c85b8, 0xc0255346f0}, 0xc04727e280, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc020b1d009, 0x8}}, 0x0, 0xc025534690, 0xc0255346c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0255346f0}, {0x643d810, 0xc01bb358c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0869fe2d0}, {0x643d810, 0xc01bb358c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0869fe2d0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219000280 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaffb31a0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04a70ce80, 0xc0e3fa3000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04a70ce80, {0xc0e3fa3000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04a70ce80, {0xc0e3fa3000, 0x735, 0xc07ce4b540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0450cbf78, {0xc0e3fa3000, 0x7eda79, 0xc0e3fa3005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc020d1f860, {0xc0e3fa3000, 0xc073952c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2690ee8, {0xc0e3fa3000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0aa5165f8, {0x631b900, 0xc0b2690ee8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0aa516380, {0x7fe42ab59310, 0xc020d1f860}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0aa516380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0aa516380, {0xc0e5b23000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02a29fa88, {0xc0e5b23000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0e5b24088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0e5b24088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0e5b24660, {0x634aac0, 0xc0e5b24088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0e5b24000, 0xc0411ee5b0, 0xc08869942b, 0xc08869942d, {0x635bec0, 0xc063de6510}, 0xc000d6ae00, 0xc055f8e448, 0xc088699429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0e5b24000, {0x63c85b8, 0xc0575a78f0}, 0xc0411ee5a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0e5b01025, 0x8}}, 0x0, 0xc0575a7890, 0xc0575a78c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0575a78f0}, {0x643d810, 0xc020d1f860}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07421ae10}, {0x643d810, 0xc020d1f860}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07421ae10}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219342749 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3df8c0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0a5b36300, 0xc036938000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0a5b36300, {0xc036938000, 0x2b88, 0x2b88}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0a5b36300, {0xc036938000, 0x2b88, 0xc04d439e00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0561aea28, {0xc036938000, 0x7eda79, 0xc036938005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0a3017d40, {0xc036938000, 0x0, 0x2b88}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2690f18, {0xc036938000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc09bb95e78, {0x631b900, 0xc0b2690f18}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc09bb95c00, {0x7fe42ab59310, 0xc0a3017d40}, 0x2b88) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc09bb95c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc09bb95c00, {0xc051b5f000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0a6f3d428, {0xc051b5f000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06073a088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06073a088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06073a660, {0x634aac0, 0xc06073a088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06073a000, 0xc0436f4830, 0xc00332942b, 0xc00332942d, {0x635bec0, 0xc09a17f950}, 0xc000d6ae00, 0xc058bba3f8, 0xc003329429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06073a000, {0x63c85b8, 0xc041112db0}, 0xc0436f4820, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc03e7d0037, 0x8}}, 0x0, 0xc041112d50, 0xc041112d80, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc041112db0}, {0x643d810, 0xc0a3017d40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01c5c2600}, {0x643d810, 0xc0a3017d40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01c5c2600}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807460 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e099000, {0x63c85b8, 0xc03c985ec0}, 0xc0987f0400, 0xc06ec13040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc03c985ec0}, 0xc0987f0400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110577c9202, 0x0, 0x0}, 0xc090a3e3c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110577c9202, 0x0, 0x0}, 0xc090a3e3c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c985e30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c985e30}, {0xc076023918, 0x660089}, 0xc076023920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c985e30}, {0xc00102e4e0}, 0xc0987f0300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c985ce0}, 0xc0987f0300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc023673d40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03c985c20}, {0xc00dfd36b0, 0x24}, 0x46c7be871, 0xc0209a1d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc02269ce18, 0xc01d6bbdb8}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc03c985c20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803537 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020f888c0, {0x63c85b8, 0xc028d88810}, 0xc06c871668, 0x1, 0xc0791310a8, 0x519bbc, 0xc04abde7e0, 0xc04a448d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc028d88810}, 0x1e0, 0xc079131230, 0x4abde780, 0x0, 0x660069, 0xc04abde7e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020f888c0, {0x63c85b8, 0xc028d88810}, 0xc04abde780, {{{{0x0, 0x0}, {0xc01e940240, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc079131448, {0x63c85b8, 0xc028d88810}, 0x203021, 0x203021) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0211060f0, {0x63c85b8, 0xc028d88810}, {0xc05928c7e0, {0x174301102d308711, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0211060f0, {0x63c85b8, 0xc028d88810}, 0xc057ce12c0, 0xc0791319f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc028d88810}, 0x660089, {0xc05928c7e0, {0x174301102d308711, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c0400, {0x63c85b8, 0xc028d88810}, 0xc06650f800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d308711, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d308711, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc028d886c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028d886c0}, {0xc0791333f8, 0x660089}, 0xc079133400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc028d886c0}, {0xc0015221e0}, 0xc06650f600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc028d88570}, 0xc06650f600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}, 0xc0204920a0, 0xc03ce97020) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}, 0x3, 0xc06beb31c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079133980}}}, {0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}, 0xc0204920a0, 0xc06beb31c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc028d88570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028d88570}, {0x40, 0xc079133b70}, 0xc079133ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc028d88570}, {0x4e9d7a0, 0xc06650f600}, 0xc0204920a0, 0xc03ce97020) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc028d88570}, 0xc01ffcca80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05928c6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05928c6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219798717 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc08923c548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc08923c5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0b1ede0f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc05f784700, {0xc0a3ec00a0, 0x2b, 0x45}, {0xc0b1afc770, 0x2b, 0x6d}, 0x1, 0xe0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0b1eddf88, {0xc0a3ec00a0, 0x1, 0x6d}, {0xc0b1afc770, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b1eddf88, {0xc0a3ec00a0, 0x2b, 0x45}, {0xc0b1afc770, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b1edd6c0, {0xc0b1afc770, 0x6d, 0x45}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b1edd6c0, {0xc0a3ec00a0, 0x2b, 0xc0b1afc770}, {0xc0b1afc770, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b1edd400, {0xc0b1afc770, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc057172008, {{0xc063b146f0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc059e97100, {{0xc063b146f0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc045cf3800, {0x63c85b8, 0xc06e8a9530}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06e8a9530}, {0x64a6560, 0xc059e97100}, {0xc063b146f0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06e8a9530}, {0x6466ed8, 0xc057172000}, {0xc063b146f0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c7ea378}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0841d7300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020578000, {0x63c85b8, 0xc06e8a9530}, 0xc0841d7300, 0xc02ddaa1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020578000, {0x63c85b8, 0xc06e8a9530}, 0xc0841d7300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e8a94a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e8a94a0}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e8a94a0}, {0xc00102e4e0}, 0xc0841d7200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e8a9440}, 0xc0841d7200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03c4c6ba0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802145 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020c29180, {0x63c85b8, 0xc02de814a0}, 0xc03f7487b0, 0x1, 0x0, 0x1, 0xc0530c9140, 0xc03da816e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02de814a0}, 0x120, 0xc05922c728, 0x530c90e0, 0x0, 0x660069, 0xc0530c9140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020c29180, {0x63c85b8, 0xc02de814a0}, 0xc0530c90e0, {{{{0x0, 0x0}, {0xc051d386c0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02de814a0}, 0x6352410, 0x561af648) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc020a61630, {0x63c85b8, 0xc02de814a0}, {0xc06062c000, {0x17430110234b556f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc020a61630, {0x63c85b8, 0xc02de814a0}, 0xc045654140, 0xc07c43eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02de814a0}, 0x660089, {0xc06062c000, {0x17430110234b556f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c61400, {0x63c85b8, 0xc02de814a0}, 0xc06b9a2800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110234b556f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110234b556f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02de81020}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02de81020}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02de81020}, {0xc00102e4e0}, 0xc06b9a2700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02de80f90}, 0xc06b9a2700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0450a79e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808070 [runnable]: internal/poll.runtime_pollWait(0x7fdc6af29ac8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc09201c000, 0xc03d52f000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc09201c000, {0xc03d52f000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc09201c000, {0xc03d52f000, 0x735, 0xc05c1d28c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc032430d40, {0xc03d52f000, 0x7eda79, 0xc03d52f005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc08e58d980, {0xc03d52f000, 0x35318b4, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc027bebf80, {0xc03d52f000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0b9715778, {0x631b900, 0xc027bebf80}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0b9715500, {0x7fe42ab59310, 0xc08e58d980}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0b9715500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0b9715500, {0xc09fdeb000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0258043c0, {0xc09fdeb000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc006f4cb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc006f4cb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc006f4d0e0, {0x634aac0, 0xc006f4cb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc006f4ca80, 0xc03c3a08c0, 0xc07940742b, 0xc07940742d, {0x635bec0, 0xc03ea4de60}, 0xc000d6ae00, 0xc01bb40e78, 0xc079407429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc006f4ca80, {0x63c85b8, 0xc0712b3b90}, 0xc03c3a08b0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc003cff009, 0x8}}, 0x0, 0xc0712b3b30, 0xc0712b3b60, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0712b3b90}, {0x643d810, 0xc08e58d980}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0321023c0}, {0x643d810, 0xc08e58d980}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0321023c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802957 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024ad3c00, {0x63c85b8, 0xc0136e7650}, 0xc0b91cd900, 0xc05c528320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024ad3c00, {0x63c85b8, 0xc0136e7650}, 0xc0b91cd900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011028d399f9, 0x0, 0x0}, 0xc08fd7fd60, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011028d399f9, 0x0, 0x0}, 0xc08fd7fd60, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0136e7350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0136e7350}, {0xc06ce813f8, 0x660089}, 0xc06ce81400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0136e7350}, {0xc03f137300}, 0xc0b91cd700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0136e7110}, 0xc0b91cd700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}, 0xc01f1a82c0, 0xc0af5e08e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}, 0x3, 0xc0116a2800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06ce81980}}}, {0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}, 0xc01f1a82c0, 0xc0116a2800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0136e7110}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0136e7110}, {0x40, 0xc06ce81b70}, 0xc06ce81ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0136e7110}, {0x4e9d7a0, 0xc0b91cd700}, 0xc01f1a82c0, 0xc0af5e08e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0136e7110}, 0xc06ae50780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05f6d0b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05f6d0b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812457 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020f888c0, {0x63c85b8, 0xc022c19770}, 0xc0c05c3188, 0x1, 0xc03a21b0a8, 0x519bbc, 0xc00b133b60, 0xc01e35d920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc022c19770}, 0x120, 0xc03a21b230, 0xb133b00, 0x0, 0x660069, 0xc00b133b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020f888c0, {0x63c85b8, 0xc022c19770}, 0xc00b133b00, {{{{0x0, 0x0}, {0xc0589299e0, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc022c19770}, 0x6352410, 0x47c52438) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0211060f0, {0x63c85b8, 0xc022c19770}, {0xc00b1338c0, {0x17430110802c6749, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0211060f0, {0x63c85b8, 0xc022c19770}, 0xc0977be320, 0xc03a21b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022c19770}, 0x660089, {0xc00b1338c0, {0x17430110802c6749, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c0400, {0x63c85b8, 0xc022c19770}, 0xc08256ab00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110802c6749, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110802c6749, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022c19680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022c19680}, {0xc03a21d3f8, 0x660089}, 0xc03a21d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022c19680}, {0x7fde1808e4a0}, 0xc08256a900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022c19590}, 0xc08256a900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}, 0xc01e517b80, 0xc0c05c31e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}, 0x3, 0xc03d2c9800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03a21d980}}}, {0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}, 0xc01e517b80, 0xc03d2c9800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022c19590}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022c19590}, {0x40, 0xc03a21db70}, 0xc03a21dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022c19590}, {0x4e9d7a0, 0xc08256a900}, 0xc01e517b80, 0xc0c05c31e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022c19590}, 0xc0c71f56e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0a9514000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0a9514000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812768 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809108 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc04d873260}, 0xc015796918, 0x1, 0x0, 0x1, 0xc04caea3c0, 0xc06acbe2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04d873260}, 0x120, 0xc020cd0728, 0x4caea360, 0x0, 0x660069, 0xc04caea3c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc04d873260}, 0xc04caea360, {{{{0x0, 0x0}, {0xc031d41440, 0x2a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d873260}, 0x6352410, 0x4b1bc0d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc04d873260}, {0xc073fcc360, {0x1743011061c0e40f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc04d873260}, 0xc04c6ff7c0, 0xc06f204ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d873260}, 0x660089, {0xc073fcc360, {0x1743011061c0e40f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc04d873260}, 0xc041b4dc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061c0e40f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061c0e40f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d8731d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d8731d0}, {0xc0025148f0, 0x660089}, 0xc0025148f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d8731d0}, {0xc00102e4e0}, 0xc041b4db00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d873140}, 0xc041b4db00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc041e10bd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804890 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc060d4c190, 0x2b) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc056d26a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc056d270e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05c676c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05c676c00, {0x63c8510, 0xc060d4c1c0}, 0xc0829c7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04c7cc8b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc060d4c1c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04c7cc8b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801415 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f22b7c0, {0x63c85b8, 0xc067188090}, 0xc00065bda0, 0x1, 0xc0065250a8, 0x519bbc, 0xc04e8c0060, 0xc041577380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc067188090}, 0x120, 0xc006525230, 0x4e8c0000, 0x0, 0x660069, 0xc04e8c0060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f22b7c0, {0x63c85b8, 0xc067188090}, 0xc04e8c0000, {{{{0x0, 0x0}, {0xc0639cf9e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc067188090}, 0x6352410, 0x1a06e220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f219d60, {0x63c85b8, 0xc067188090}, {0xc04fca1e60, {0x1743011018bca093, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f219d60, {0x63c85b8, 0xc067188090}, 0xc05f31c5a0, 0xc0065259f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067188090}, 0x660089, {0xc04fca1e60, {0x1743011018bca093, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f6c7800, {0x63c85b8, 0xc067188090}, 0xc06308c800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011018bca093, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011018bca093, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067188000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067188000}, {0xc0065273f8, 0x660089}, 0xc006527400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067188000}, {0xc0015221e0}, 0xc06308c600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01a4efe30}, 0xc06308c600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}, 0xc0137dc960, 0xc026e965e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}, 0x3, 0xc08714fd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006527980}}}, {0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}, 0xc0137dc960, 0xc08714fd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01a4efe30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01a4efe30}, {0x40, 0xc006527b70}, 0xc006527ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01a4efe30}, {0x4e9d7a0, 0xc06308c600}, 0xc0137dc960, 0xc026e965e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01a4efe30}, 0xc0450baba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037336120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037336120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812362 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc09d182850, {0x63c85b8, 0xc06c1f4420}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc06c1f4420}, {0x4f67ad2, 0x6446ff8}, 0xc002c9efb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219811919 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0185643c0, {0x63c85b8, 0xc02178c540}, 0xc06c870f00, 0x1, 0xc01654b0a8, 0x519bbc, 0xc0708b4960, 0xc0d0fe0060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02178c540}, 0x120, 0xc01654b230, 0x708b4900, 0x0, 0x660069, 0xc0708b4960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0185643c0, {0x63c85b8, 0xc02178c540}, 0xc0708b4900, {{{{0x0, 0x0}, {0xc04c615320, 0xf}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02178c540}, 0x6352230, 0x3c90c4f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0185322d0, {0x63c85b8, 0xc02178c540}, {0xc0708b47e0, {0x17430110836db57a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0185322d0, {0x63c85b8, 0xc02178c540}, 0xc079989400, 0xc01654b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02178c540}, 0x660089, {0xc0708b47e0, {0x17430110836db57a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018570c00, {0x63c85b8, 0xc02178c540}, 0xc02664e800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110836db57a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110836db57a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02178c300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02178c300}, {0xc01654d3f8, 0x660089}, 0xc01654d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02178c300}, {0x7fe423ff01f0}, 0xc031f85f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02178c270}, 0xc031f85f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}, 0xc03d9acdc0, 0xc023415230) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}, 0x3, 0xc048fde280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01654d980}}}, {0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}, 0xc03d9acdc0, 0xc048fde280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02178c270}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02178c270}, {0x40, 0xc01654db70}, 0xc01654dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02178c270}, {0x4e9d7a0, 0xc031f85f00}, 0xc03d9acdc0, 0xc023415230) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02178c270}, 0xc048f9f7a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0708b45a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0708b45a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809229 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b9a2b40, {0x63c85b8, 0xc01618c420}, 0xc04a225ab8, 0x1, 0xc01e6b70a8, 0x519bbc, 0xc05330b020, 0xc04e3f55c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01618c420}, 0x120, 0xc01e6b7230, 0x5330afc0, 0x0, 0x660069, 0xc05330b020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b9a2b40, {0x63c85b8, 0xc01618c420}, 0xc05330afc0, {{{{0x0, 0x0}, {0xc059b9a120, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01618c420}, 0x6352410, 0x42a0f028) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0fb950, {0x63c85b8, 0xc01618c420}, {0xc05330aea0, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0fb950, {0x63c85b8, 0xc01618c420}, 0xc070dc0500, 0xc01e6b79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01618c420}, 0x660089, {0xc05330aea0, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9bb000, {0x63c85b8, 0xc01618c420}, 0xc0523d4a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01618c360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01618c360}, {0xc01e6b93f8, 0x660089}, 0xc01e6b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01618c360}, {0x632b4f5161544c44}, 0xc0523d4800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01618c270}, 0xc0523d4800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}, 0xc02d9d6ba0, 0xc043945140) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}, 0x3, 0xc03f78b600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01e6b9980}}}, {0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}, 0xc02d9d6ba0, 0xc03f78b600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01618c270}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01618c270}, {0x40, 0xc01e6b9b70}, 0xc01e6b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01618c270}, {0x4e9d7a0, 0xc0523d4800}, 0xc02d9d6ba0, 0xc043945140) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01618c270}, 0xc0242a34a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330aa20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330aa20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801119 [runnable]: sync.runtime_SemacquireMutex(0xc049819198, 0x8f, 0x7fd4c2bf64c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc049819040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc049819068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04ccbe000, {0xffffffffffffff15, 0xc0498191c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc04ccbe000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04ccbe000, {0xc08275d080, 0x32, 0xb0}, {0xc02b8dab80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07cbd2d98, {0xc08275d080, 0x32, 0xb0}, {0xc02b8dab80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07cbd16c0, {0xc02b8dab80, 0x74, 0xb0}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07cbd16c0, {0xc08275d080, 0x74, 0xc02b8dab80}, {0xc02b8dab80, 0xc07146f340, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07cbd1400, {0xc02b8dab80, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0428f8da8, {{0xc07146f340, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0579e8f00, {{0xc0531cb350, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0520f2000, {0x63c85b8, 0xc063b9d980}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc063b9d980}, {0x64a6560, 0xc0579e8f00}, {0xc0531cb350, 0xc00bc1fa40, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc063b9d980}, {0x6466ed8, 0xc0428f8b00}, {0xc0531cb350, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06edf1f80}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc051024700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c5c000, {0x63c85b8, 0xc063b9d980}, 0xc051024700, 0xc05ed41860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c5c000, {0x63c85b8, 0xc063b9d980}, 0xc051024700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc063b9d8f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063b9d8f0}, {0xc04981f3f8, 0x660089}, 0xc04981f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc063b9d8f0}, {0x7fe3e21cf390}, 0xc051024400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc063b9d860}, 0xc051024400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}, 0xc0466a33a0, 0xc0323e9b48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}, 0x3, 0xc093ed9300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04981f980}}}, {0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}, 0xc0466a33a0, 0xc093ed9300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc063b9d860}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063b9d860}, {0x40, 0xc04981fb70}, 0xc04981fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc063b9d860}, {0x4e9d7a0, 0xc051024400}, 0xc0466a33a0, 0xc0323e9b48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc063b9d860}, 0xc03b1866c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0896bb200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0896bb200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795545 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc04d6facf0}, 0xc064a3a100, 0xc07b09b4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc04d6facf0}, 0xc064a3a100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c82f06f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c82f06f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d6fac60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d6fac60}, {0xc07bae87f8, 0x660089}, 0xc07bae8800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d6fac60}, {0x24}, 0xc064a3a000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d6fac00}, 0xc064a3a000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0a1879b80, {0x63c85b8, 0xc04d6faab0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc03fc50600, {0x63c85b8, 0xc04d6faab0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc03fc50600, {0x63c8510, 0xc0482acf40}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc03fc50600, {0x63c8510, 0xc0482acf40}, {0x63f7db8, 0x9bf6f48}, 0xc07baed088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc03fc50600, {0x63c85b8, 0xc066a754d0}, {{0x63f7db8, 0x9bf6f48}, {0xc0b8ce2b5b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc03fc50600, {0x63c85b8, 0xc066a754d0}, {{0x63f7db8, 0x9bf6f48}, {0xc0b8ce2b5b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0b8ce2b5b, 0x6}, 0x0, 0x0}, {0x318adebc, 0xedb7a834f, 0x0}, {0x318ae496, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03fc50600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03fc50600, {0x63c8510, 0xc091e21ac0}, 0xc0768eddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07b27d2c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc091e21ac0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07b27d2c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219796686 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0ad867620}, 0xc065ff4cb0, {0x646cc48, 0xc0ad867680}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0ad867620}, 0xedb7a834e, 0xc05ff8b8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0ad867620}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc01e298510}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc065ff4ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc01e298510}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807312 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03005e780, {0x63c85b8, 0xc041b56450}, 0xc0735b0b88, 0x1, 0xc0038ef0a8, 0x519bbc, 0xc07d348060, 0xc06e4b3320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc041b56450}, 0x1e0, 0xc0038ef230, 0x7d348000, 0x0, 0x660069, 0xc07d348060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03005e780, {0x63c85b8, 0xc041b56450}, 0xc07d348000, {{{{0x0, 0x0}, {0xc046efc5a0, 0xf}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc041b56450}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030007040, {0x63c85b8, 0xc041b56450}, {0xc089ba2360, {0x1743011057370ae8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030007040, {0x63c85b8, 0xc041b56450}, 0xc04def8be0, 0xc0038ef9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041b56450}, 0x660089, {0xc089ba2360, {0x1743011057370ae8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc041b56450}, 0xc08f7c1000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057370ae8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057370ae8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041b563c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041b563c0}, {0xc0038f13f8, 0x660089}, 0xc0038f1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041b563c0}, {0x3fefffff7865836b}, 0xc08f7c0e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041b56330}, 0xc08f7c0e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}, 0xc0625077e0, 0xc06c49a210) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}, 0x3, 0xc0792e6f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0038f1980}}}, {0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}, 0xc0625077e0, 0xc0792e6f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041b56330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041b56330}, {0x40, 0xc0038f1b70}, 0xc0038f1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041b56330}, {0x4e9d7a0, 0xc08f7c0e00}, 0xc0625077e0, 0xc06c49a210) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041b56330}, 0xc04b10b020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc089ba2240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc089ba2240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808036 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0615b7d50, {0x63c85b8, 0xc0709e2c30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0709e2c30}, {0x4f67ad2, 0x6446ff8}, 0xc01f3f2fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219811170 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc03531adb0}, 0xc02d8dc420, 0x1, 0xc026a670a8, 0x519bbc, 0xc0395d9ec0, 0xc0434b5920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03531adb0}, 0x120, 0xc026a67230, 0x395d9e60, 0x0, 0x660069, 0xc0395d9ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc03531adb0}, 0xc0395d9e60, {{{{0x0, 0x0}, {0xc0b70a2d80, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03531adb0}, 0x6352410, 0x412adb40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc03531adb0}, {0xc0395d9d40, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc03531adb0}, 0xc024ef8e60, 0xc026a679f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03531adb0}, 0x660089, {0xc0395d9d40, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc03531adb0}, 0xc07d06f300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03531ad20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03531ad20}, {0xc026a693f8, 0x660089}, 0xc026a69400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03531ad20}, {0x7fe3f1fa2800}, 0xc07d06f100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03531ac90}, 0xc07d06f100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}, 0xc01544a8e0, 0xc0a7c03ba8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}, 0x3, 0xc0c1704140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026a69980}}}, {0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}, 0xc01544a8e0, 0xc0c1704140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03531ac90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03531ac90}, {0x40, 0xc026a69b70}, 0xc026a69ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03531ac90}, {0x4e9d7a0, 0xc07d06f100}, 0xc01544a8e0, 0xc0a7c03ba8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03531ac90}, 0xc01c739f20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0395d9c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0395d9c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809211 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc0213826f0}, 0xc04b7368d0, 0x1, 0xc00653dff0, 0x519bbc, 0xc0433e4ba0, 0xc01ea2a240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0213826f0}, 0x1e0, 0xc00653e178, 0x433e4b40, 0x0, 0x660069, 0xc0433e4ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc0213826f0}, 0xc0433e4b40, {{{{0x0, 0x0}, {0xc0abac98c0, 0x5b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0213826f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc0213826f0}, {0xc05aa1f200, {0x1743011063a69dc2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc0213826f0}, 0xc05c702c80, 0xc00b2c0938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0213826f0}, 0x660089, {0xc05aa1f200, {0x1743011063a69dc2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0213826f0}, 0xc05f8ce700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063a69dc2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063a69dc2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ca4df20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ca4df20}, {0xc00b2c2340, 0x660089}, 0xc00b2c2348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ca4df20}, {0xc00102e4e0}, 0xc05f8ce600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ca4dd70}, 0xc05f8ce600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02ca4c1b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219150585 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00e9692c0, {0xc045e68f28, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00e9692c0, {0xc045e68f28, 0xc0361709f0, 0xc007b5aa68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc06893d920, {0xc045e68f28, 0xc007b5aae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc06893d920}, {0xc045e68f28, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc06521d8c0, {0xc045e68f28, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc045e68f18, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0x0, 0xc06521d8c0, {0x0, 0x0}, 0x7fffffff, 0xc007b5ad90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0x0, {0x7fe42ab594c0, 0xc000808f40}, 0x0, {0x0, 0x0}, {0x4cfe5a0, 0xc04a6d9b78}, 0x1e0000001e, 0xc007b5ad90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc007d30680, {0x4cfe5a0, 0xc04a6d9b78}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0a1948380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc0a1948380}, 0xc0b41a6fb0, {0x63c85b8, 0xc06f4bed20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc06f4bed20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812798 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802727 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01dd1a800, {0x63c85b8, 0xc02480bd70}, 0xc03dc3cf00, 0xc02eccde00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd1a800, {0x63c85b8, 0xc02480bd70}, 0xc03dc3cf00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025dec344, 0x0, 0x0}, 0xc092783a60, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025dec344, 0x0, 0x0}, 0xc092783a60, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02480bce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02480bce0}, {0xc065a9f3f8, 0x660089}, 0xc065a9f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02480bce0}, {0xc0015221e0}, 0xc03dc3cd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02480bc20}, 0xc03dc3cd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}, 0xc04f1ae000, 0xc02747d8a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}, 0x3, 0xc0118adf40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc065a9f980}}}, {0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}, 0xc04f1ae000, 0xc0118adf40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02480bc20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02480bc20}, {0x40, 0xc065a9fb08}, 0xc065a9fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02480bc20}, {0x4e9d7a0, 0xc03dc3cd00}, 0xc04f1ae000, 0xc02747d8a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02480bc20}, 0xc04702d740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04c2b10e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04c2b10e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809808 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc01cdd5dd0}, 0xc072f963a8, 0x1, 0xc00a4450a8, 0x519bbc, 0xc08979d800, 0xc059ee7aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0189dd020, {0x63c85b8, 0xc01cdd5dd0}, 0xc068122a20, 0xc00a4451a8, 0x8979d800, 0x0, 0x2, 0xc08979d800, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc01cdd5dd0}, 0xc019521a40, {{{{0xc079db7b00, 0x46}, {0xc035cd6b40, 0x70}}, {{0xc00449d680, 0x96}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01cdd5dd0}, 0x63521a0, 0x6b2454a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc01cdd5dd0}, {0xc04aa29200, {0x1743011004128f9a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc01cdd5dd0}, 0xc01c4d25a0, 0xc00a4459f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01cdd5dd0}, 0x660089, {0xc04aa29200, {0x1743011004128f9a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc01cdd5dd0}, 0xc06f82f500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011004128f9a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011004128f9a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01cdd5d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01cdd5d40}, {0xc00a4473f8, 0x660089}, 0xc00a447400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01cdd5d40}, {0x7fe425732870}, 0xc06f82f200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01cdd5c50}, 0xc06f82f200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}, 0xc0701d4000, 0xc0bb3a9398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}, 0x3, 0xc018d8f840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a447980}}}, {0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}, 0xc0701d4000, 0xc018d8f840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01cdd5c50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01cdd5c50}, {0x40, 0xc00a447b08}, 0xc00a447ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01cdd5c50}, {0x4e9d7a0, 0xc06f82f200}, 0xc0701d4000, 0xc0bb3a9398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01cdd5c50}, 0xc0195218c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04aa28fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04aa28fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219688170 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b26d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc08dbd7f00, 0xc015858000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc08dbd7f00, {0xc015858000, 0x2b93, 0x2b93}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc08dbd7f00, {0xc015858000, 0x2b93, 0xc075e8e3c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc02885f2c0, {0xc015858000, 0x7eda79, 0xc015858005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02c37b080, {0xc015858000, 0x0, 0x2b93}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a901e0, {0xc015858000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03b1693f8, {0x631b900, 0xc043a901e0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03b169180, {0x7fe42ab59310, 0xc02c37b080}, 0x2b93) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03b169180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03b169180, {0xc099626000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0311b6450, {0xc099626000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0db964b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0db964b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0db9650e0, {0x634aac0, 0xc0db964b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0db964a80, 0xc03fabc2b0, 0xc071ae742b, 0xc071ae742d, {0x635bec0, 0xc0617be2d0}, 0xc000d6ae00, 0xc0973c8618, 0xc071ae7429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0db964a80, {0x63c85b8, 0xc01e10c330}, 0xc03fabc2a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc03884b037, 0x8}}, 0x0, 0xc01e10c240, 0xc01e10c2d0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc01e10c330}, {0x643d810, 0xc02c37b080}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc062923950}, {0x643d810, 0xc02c37b080}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc062923950}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803832 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01ef1c000, {0x63c85b8, 0xc04ba46030}, 0xc039a33200, 0xc0627cd720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ef1c000, {0x63c85b8, 0xc04ba46030}, 0xc039a33200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103213b64e, 0x0, 0x0}, 0xc01e29ea40, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103213b64e, 0x0, 0x0}, 0xc01e29ea40, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073155f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073155f80}, {0xc0624453f8, 0x660089}, 0xc062445400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073155f80}, {0x0}, 0xc039a32f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073155ef0}, 0xc039a32f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}, 0xc072246720, 0xc07cc2bc98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}, 0x3, 0xc037bd1c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062445980}}}, {0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}, 0xc072246720, 0xc037bd1c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073155ef0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073155ef0}, {0x40, 0xc062445b70}, 0xc062445ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073155ef0}, {0x4e9d7a0, 0xc039a32f00}, 0xc072246720, 0xc07cc2bc98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073155ef0}, 0xc022a9d020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810042 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e72bc00, {0x63c85b8, 0xc033ea12f0}, 0xc0ac872b00, 0xc0415d0f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc033ea12f0}, 0xc0ac872b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bc1e6a5, 0x0, 0x0}, 0xc03cc76350, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bc1e6a5, 0x0, 0x0}, 0xc03cc76350, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc033ea1230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc033ea1230}, {0xc0766b7a08, 0x660089}, 0xc0766b7a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc033ea1230}, {0xc00102e4e0}, 0xc0ac872900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc033ea11d0}, 0xc0ac872900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc033ea1140}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc033ea1140}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812422 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0300a9c00, {0x63c85b8, 0xc050b470e0}, 0xc0c122dd00, 0xc029c98820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0300a9c00, {0x63c85b8, 0xc050b470e0}, 0xc0c122dd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108ab213f7, 0x0, 0x0}, 0xc033eaf9b0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108ab213f7, 0x0, 0x0}, 0xc033eaf9b0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f568ed0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f568ed0}, {0xc07d93f918, 0x660089}, 0xc07d93f920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f568ed0}, {0xc00102e4e0}, 0xc070375900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f568e70}, 0xc070375900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06123ad20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04f568db0}, {0xc00dfd36b0, 0x24}, 0x4689ef9aa, 0xc02790dd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc031d66420}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04f568db0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804881 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a65d400, {0x63c85b8, 0xc048382f90}, 0xc0483f72f0, 0x1, 0x708017d898f8a9, 0x37e11d600, 0xc054937320, 0xc035a2e7e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc048382f90}, 0x1e0, 0xc08606f230, 0x549372c0, 0xc03fefdba0, 0x18, 0xc054937320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a65d400, {0x63c85b8, 0xc048382f90}, 0xc0549372c0, {{{{0x0, 0x0}, {0xc059b9a240, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x652b514967373145, {0x63c85b8, 0xc048382f90}, 0xd819d2ee42170089, 0x2509d84f) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a5d7b30, {0x63c85b8, 0xc048382f90}, {0xc06f85d560, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a5d7b30, {0x63c85b8, 0xc048382f90}, 0xc087686c80, 0xc08606f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc048382f90}, 0x660089, {0xc06f85d560, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a26f400, {0x63c85b8, 0xc048382f90}, 0xc011f4e900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048382ea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048382ea0}, {0xc0860713f8, 0x660089}, 0xc086071400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048382ea0}, {0x203009}, 0xc011f4e700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048382e10}, 0xc011f4e700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}, 0xc0235c54a0, 0xc0483f72c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}, 0x3, 0xc01ae27340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc086071980}}}, {0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}, 0xc0235c54a0, 0xc01ae27340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048382e10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048382e10}, {0x40, 0xc086071b70}, 0xc086071ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}, 0x203006, 0x203006) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048382e10}, {0x4e9d7a0, 0xc011f4e700}, 0xc0235c54a0, 0xc0483f72c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048382e10}, 0xc027dfaea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85cea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85cea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800081 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc029e825d0}, 0xc0434ab410, 0x1, 0xc04cc390a8, 0x519bbc, 0xc028e8a9c0, 0xc0330c0d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc029e825d0}, 0x1e0, 0xc04cc39230, 0x28e8a960, 0x0, 0x660069, 0xc028e8a9c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc029e825d0}, 0xc028e8a960, {{{{0x0, 0x0}, {0xc041575b00, 0x55}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc029e825d0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc029e825d0}, {0xc0290439e0, {0x174301101740f520, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc029e825d0}, 0xc056c64140, 0xc04cc399f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc029e825d0}, 0x660089, {0xc0290439e0, {0x174301101740f520, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc029e825d0}, 0xc09875dd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101740f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101740f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029e82540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029e82540}, {0xc04cc3b3f8, 0x660089}, 0xc04cc3b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029e82540}, {0xc0015221e0}, 0xc09875db00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029e82480}, 0xc09875db00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}, 0xc07af73b20, 0xc070c4b6e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}, 0x3, 0xc042af8e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04cc3b980}}}, {0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}, 0xc07af73b20, 0xc042af8e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029e82480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029e82480}, {0x40, 0xc04cc3bb70}, 0xc04cc3bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029e82480}, {0x4e9d7a0, 0xc09875db00}, 0xc07af73b20, 0xc070c4b6e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029e82480}, 0xc033f78c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0290438c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0290438c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808381 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4000, {0x63c85b8, 0xc04cccb8c0}, 0xc08993aa00, 0xc07ae3bb80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4000, {0x63c85b8, 0xc04cccb8c0}, 0xc08993aa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e0af035, 0x0, 0x0}, 0xc0926d1500, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e0af035, 0x0, 0x0}, 0xc0926d1500, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04cccb830}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cccb830}, {0xc00602f3f8, 0x660089}, 0xc00602f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04cccb830}, {0x3feffffee1276011}, 0xc08993a600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04cccb740}, 0xc08993a600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}, 0xc061a4e340, 0xc06d54a570) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}, 0x3, 0xc0100dc800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00602f980}}}, {0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}, 0xc061a4e340, 0xc0100dc800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04cccb740}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cccb740}, {0x40, 0xc00602fb70}, 0xc00602fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04cccb740}, {0x4e9d7a0, 0xc08993a600}, 0xc061a4e340, 0xc06d54a570) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04cccb740}, 0xc08ce8b0e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0621a6b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0621a6b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803718 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020010800, {0x63c85b8, 0xc022bee8d0}, 0xc06d305f00, 0xc0ab9ad680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020010800, {0x63c85b8, 0xc022bee8d0}, 0xc06d305f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032101887, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032101887, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022bee7b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022bee7b0}, {0xc0790733f8, 0x660089}, 0xc079073400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022bee7b0}, {0x0}, 0xc06d305c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022bee6f0}, 0xc06d305c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}, 0xc006be1a40, 0xc025fee168) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}, 0x3, 0xc03692f580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079073980}}}, {0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}, 0xc006be1a40, 0xc03692f580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022bee6f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022bee6f0}, {0x40, 0xc079073b70}, 0xc079073ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022bee6f0}, {0x4e9d7a0, 0xc06d305c00}, 0xc006be1a40, 0xc025fee168) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022bee6f0}, 0xc033c10000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057b89680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057b89680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803105 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc07d081d40}, 0xc06f9e4c00, 0xc0307a7ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07d081d40}, 0xc06f9e4c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032129bc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032129bc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07d081cb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07d081cb0}, {0xc09630e7f8, 0x660089}, 0xc09630e800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07d081cb0}, {0xc00102e4e0}, 0xc06f9e4a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07d081bf0}, 0xc06f9e4a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc066ea9080, {0x63c85b8, 0xc07d081aa0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc06e21f900, {0x63c85b8, 0xc07d081aa0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc06e21f900, {0x63c8510, 0xc032b993c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc06e21f900, {0x63c8510, 0xc032b993c0}, {0x63f7db8, 0x9bf6f48}, 0xc096313088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc06e21f900, {0x63c85b8, 0xc073155dd0}, {{0x63f7db8, 0x9bf6f48}, {0xc0525df318, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc06e21f900, {0x63c85b8, 0xc073155dd0}, {{0x63f7db8, 0x9bf6f48}, {0xc0525df318, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0525df318, 0x6}, 0x0, 0x0}, {0x1766b996, 0xedb7a834f, 0x0}, {0x1766c29b, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06e21f900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06e21f900, {0x63c8510, 0xc03d2c1940}, 0xc075e55dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc040f8e7d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03d2c1940}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc040f8e7d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219796499 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc06e4ca120) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc093f71130, {0x4d83cc0, 0xc05adede80}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x176) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc04b38fe60, 0xc0750f2fa8, 0xc0750f2f78) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc04b38fe60, {0x4d83cc0, 0xc05adede80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc033ea0090}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc0ac872700}, {0x4d83cc0, 0xc05adede80}, 0x2, {0xc00c5825c0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc033ea0090}, {0x4f6406d, 0x7fe457d80878}, {0x4e9d7a0, 0xc0ac872700}, {0x4d83cc0, 0xc05adede80}, 0x4b7ca9f4a7e481e1, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x7cb88dec5cd16583, {0x63c85b8, 0xc033ea0090}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc0ac872700}, {0x4d83cc0, 0xc05adede80}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc043a28c18, {0x63c85b8, 0xc033ea0090}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc043a28c18}}, {0x63c85b8, 0xc033ea0090}, 0xc0ac872700, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1b, 0x1a, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc01aa353b0, {0x63c85b8, 0xc033ea0030}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc01aa353b0, {0x63c85b8, 0xc033ea0030}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc05878fb80, {0x63c85b8, 0xc033ea0030}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc096804700, {0x63c85b8, 0xc033ea0030}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0482dc1e0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc05095e4c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc05754a720) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).next(0xc04e104420) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:374 +0x7d github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc04e104420) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:189 +0x32c github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc05095e540) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0aea92900) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc063b4f970, {0x63c8510, 0xc05095e480}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc075384c30, {0x63c8510, 0xc05095e480}, 0xc093f70bb0) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc00dfa1bc0}, 0xc0670bd6c0, 0xc093f70bb0, 0xc05adedc80, 0xc01857b180, 0xc0033682e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc00dfa1bc0}, 0xc003367ff0, 0xc0670bd6c0, 0xc036df7bc0, {{0x63ca3e8, 0xc05adedc00}, 0x0}, 0xc01857b180) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc003367900, {0x63c8510, 0xc00dfa1bc0}, 0xc003367ff0, 0xc00dfa1bc0, {0x7fe42b790cf0, 0xc036df7bc0}, 0x0, 0xc05754a718) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc003367900, {0x63c8510, 0xc00dfa1bc0}, 0xc003367ff0, {0x7fe42b790cf0, 0xc036df7bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc003367900, {0x63c85b8, 0xc043ee5110}, {{0x63f8d78, 0xc05878ed20}, {0xc05bab6e87, 0x70}, 0x2, 0x1}, 0xc04b38e6c0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc003367900, {0x63c85b8, 0xc043ee5110}, {{0x63f8d78, 0xc05878ed20}, {0xc05bab6e87, 0x70}, 0x2, 0x1}, 0xc04b38e6c0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc003367900, {0x63c85b8, 0xc043ee5110}, {0xc04b38e6c0, {0xc06e07f240, 0x2, 0x2}, {0xc0652d9520, 0xe, 0xe}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc05bab6f8d, 0x0}, 0x0, {0x14cbbab6, 0xedb7a8350, 0x0}, 0x1}, 0xc003367900, 0xc0750f9980, 0xc0750f9970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003367900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003367900, {0x63c8510, 0xc08a5ab1c0}, 0xc07658bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01e8c6470) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc08a5ab1c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01e8c6470) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 218825017 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0468416b0, {0x63c85b8, 0xc0452dd230}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0336b9900, {0x63c85b8, 0xc0452dd230}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0336b9900, {0x63c8510, 0xc037ff8900}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc0336b9900, {0x63c8510, 0xc037ff8900}, 0xc081ccddc0, {0x7fe42b790cf0, 0xc06f8e26c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0336b9900, {0x63c85b8, 0xc06c28f500}, {{0x63f88f8, 0xc081ccddc0}, {0xc0b42073a3, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0336b9900, {0x63c85b8, 0xc06c28f500}, {{0x63f88f8, 0xc081ccddc0}, {0xc0b42073a3, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc081ccddc0}, {0xc0b42073a3, 0x23}, 0x0, 0x0}, {0x1a6a5107, 0xedb7a8350, 0x0}, {0x1a6a564f, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0336b9900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0336b9900, {0x63c8510, 0xc03d1eac80}, 0xc0873afdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc040550f80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03d1eac80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc040550f80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801474 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0310be400, {0x63c85b8, 0xc04eb46de0}, 0xc014ed0200, 0xc03b365360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310be400, {0x63c85b8, 0xc04eb46de0}, 0xc014ed0200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c43ac2d, 0x0, 0x0}, 0xc030ca38b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c43ac2d, 0x0, 0x0}, 0xc030ca38b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04eb46d50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04eb46d50}, {0xc076589918, 0x660089}, 0xc076589920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04eb46d50}, {0xc00102e4e0}, 0xc014ed0100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04eb46cf0}, 0xc014ed0100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc08819daa0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04eb46bd0}, {0xc00dfd36b0, 0x24}, 0x46c7c2b83, 0xc0918d9d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc065c1fcb0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04eb46bd0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806181 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc0709197d0}, 0xc054753170, 0x1, 0xc006e350a8, 0x519bbc, 0xc04afc9ec0, 0xc04afc8720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0709197d0}, 0x120, 0xc006e35230, 0x4afc9e60, 0x0, 0x660069, 0xc04afc9ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc0709197d0}, 0xc04afc9e60, {{{{0x0, 0x0}, {0xc04bfded80, 0x24}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0709197d0}, 0x6352410, 0x1a9e31c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc0709197d0}, {0xc04afc9d40, {0x174301104be44f22, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc0709197d0}, 0xc07c6d3ea0, 0xc006e359f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0709197d0}, 0x660089, {0xc04afc9d40, {0x174301104be44f22, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc0709197d0}, 0xc0530eea00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104be44f22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104be44f22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070919740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070919740}, {0xc006e373f8, 0x660089}, 0xc006e37400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070919740}, {0xc0015221e0}, 0xc0530ee800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0709196b0}, 0xc0530ee800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}, 0xc0834a3460, 0xc08d4a8738) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}, 0x3, 0xc042211bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006e37980}}}, {0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}, 0xc0834a3460, 0xc042211bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0709196b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0709196b0}, {0x40, 0xc006e37b70}, 0xc006e37ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0709196b0}, {0x4e9d7a0, 0xc0530ee800}, 0xc0834a3460, 0xc08d4a8738) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0709196b0}, 0xc03b50dec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04afc9c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04afc9c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 202012962 [select, 31 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 219804891 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0300a9c00, {0x63c85b8, 0xc077fef710}, 0xc028dbd100, 0xc08e181e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0300a9c00, {0x63c85b8, 0xc077fef710}, 0xc028dbd100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011002c20ba8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011002c20ba8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077fef680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077fef680}, {0xc058161a28, 0x660089}, 0xc058161a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077fef680}, {0xc00102e4e0}, 0xc028dbd000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077fef5f0}, 0xc028dbd000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc071679d50, {0x63c85b8, 0xc077fef560}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc071679d50, {0x63c85b8, 0xc077fef560}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc077fef560}, {0x4f67ad2, 0xc060d4c1c0}, 0xc01646dfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219799946 [runnable]: sync.runtime_SemacquireMutex(0xc091ae0690, 0x8f, 0x9bb1080) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc091ae0538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafb8, 0xc091ae0560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0600d5200, {0xffffffffffffff15, 0xc091ae06c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0600d5200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0600d5200, {0xc01c513200, 0x32, 0x74}, {0xc016048500, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc03f310d98, {0xc01c513200, 0x32, 0x74}, {0xc016048500, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc03f30f6c0, {0xc016048500, 0x74, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc03f30f6c0, {0xc01c513200, 0x74, 0xc016048500}, {0xc016048500, 0xc01d6a51a0, 0x31}, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc03f30f400, {0xc016048500, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc05ba583a8, {{0xc01d6a51a0, 0x31, 0x60}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc045996200, {{0xc042b77620, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08d652c00, {0x63c85b8, 0xc02c9ba270}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02c9ba270}, {0x64a6560, 0xc045996200}, {0xc042b77620, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02c9ba270}, {0x6466ed8, 0xc05ba58100}, {0xc042b77620, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc066a04ba0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc059168b00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209ee000, {0x63c85b8, 0xc02c9ba270}, 0xc059168b00, 0xc06891ae60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209ee000, {0x63c85b8, 0xc02c9ba270}, 0xc059168b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02c9ba1b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c9ba1b0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02c9ba1b0}, {0xc00102e4e0}, 0xc059168a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02c9ba150}, 0xc059168a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc024122120}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791389 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02d547680}, 0xc00b036cb0, {0x646cc48, 0xc02d5476e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02d547680}, 0xedb7a834e, 0xc09ecba480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02d547680}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc044226e40}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc00b036ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc044226e40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219787899 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab97310, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc00cdc5f00, 0xc0265a3900, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00cdc5f00, {0xc0265a3900, 0x1114, 0x1114}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00cdc5f00, {0xc0265a3900, 0x1114, 0xc04fbd92c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06b245e00, {0xc0265a3900, 0xc0, 0xbf}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc05c1f52c0, {0xc0265a3900, 0x2, 0x1114}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc021b9b1e8, {0xc0265a3900, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc050f3fe78, {0x631b900, 0xc021b9b1e8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc050f3fc00, {0x7fe42ab59310, 0xc05c1f52c0}, 0x1114) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc050f3fc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc050f3fc00, {0xc0320d6000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc020562738, {0xc0320d6000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a0fa2b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a0fa2b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a0fa30e0, {0x634aac0, 0xc0a0fa2b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a0fa2a80, 0xc02d739610, 0xc01002542b, 0xc01002542d, {0x635bec0, 0xc03eeb2a20}, 0xc000d6ae00, 0xc067d795d8, 0xc010025429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a0fa2a80, {0x63c85b8, 0xc07b0b82d0}, 0xc02d739600, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc099fde036, 0x8}}, 0x0, 0xc07b0b81b0, 0xc07b0b81e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc07b0b82d0}, {0x643d810, 0xc05c1f52c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04cdc0b10}, {0x643d810, 0xc05c1f52c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04cdc0b10}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812018 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc035375a70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809540 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc0720a7aa0}, 0xc061064b00, 0xc03c3ef040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0720a7aa0}, 0xc061064b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065e1cd43, 0x0, 0x0}, 0xc04d583d90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065e1cd43, 0x0, 0x0}, 0xc04d583d90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0720a7a10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0720a7a10}, {0xc01af933f8, 0x660089}, 0xc01af93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0720a7a10}, {0xc0015221e0}, 0xc061064900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0720a7980}, 0xc061064900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}, 0xc04cf66400, 0xc00beecff0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}, 0x3, 0xc058bed2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01af93980}}}, {0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}, 0xc04cf66400, 0xc058bed2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0720a7980}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0720a7980}, {0x40, 0xc01af93b70}, 0xc01af93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0720a7980}, {0x4e9d7a0, 0xc061064900}, 0xc04cf66400, 0xc00beecff0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0720a7980}, 0xc074753bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0888f19e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0888f19e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219796288 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc00cd86580, {0x63c85b8, 0xc064734ae0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc088df9300, {0x63c85b8, 0xc064734ae0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc088df9300, {0x63c8510, 0xc03124ba40}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc088df9300, {0x63c8510, 0xc03124ba40}, {0x63f7db8, 0x9bf6f48}, 0xc07d69f088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc088df9300, {0x63c85b8, 0xc02ced4f60}, {{0x63f7db8, 0x9bf6f48}, {0xc00d85b34a, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc088df9300, {0x63c85b8, 0xc02ced4f60}, {{0x63f7db8, 0x9bf6f48}, {0xc00d85b34a, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc00d85b34a, 0x6}, 0x0, 0x0}, {0x260b221e, 0xedb7a834f, 0x0}, {0x260b27c3, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc088df9300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc088df9300, {0x63c8510, 0xc062c592c0}, 0xc0768f1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08aba2bc0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc062c592c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08aba2bc0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808211 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc07b509290}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 203350753 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc03692f410, 0x4a4) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc03281d500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc03281db60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00d1b0000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00d1b0000, {0x63c8510, 0xc03692f440}, 0xc0482a3dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0795a0dc0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03692f440}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0795a0dc0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812724 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0d05e4000, 0xc0392e7168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0d05e4000, {0xc01c46fd80, 0x27, 0x72}, {0xc034f81f80, 0x27, 0x6d}, 0x0, 0xbb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc076e6ba88, {0xc01c46fd80, 0x1, 0x6d}, {0xc034f81f80, 0xc076e6af00, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc076e6ba88, {0xc01c46fd80, 0x27, 0x72}, {0xc034f81f80, 0x27, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc076e6b1c0, {0xc034f81f80, 0x0, 0xc0392e74e0}, 0x121ea30, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc076e6b1c0, {0xc01c46fd80, 0x27, 0x6d}, {0xc034f81f80, 0x26, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc076e6af00, {0xc034f81f80, 0x27, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc058883188, {{0xc051de1920, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc027afed00, {{0xc051de1920, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ff38800, {0x63c85b8, 0xc062cde450}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062cde450}, {0x64a6560, 0xc027afed00}, {0xc051de1920, 0xc026f3fd40, 0x30}, {0x174301109322d676, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062cde450}, {0x6466ed8, 0xc058883180}, {0xc051de1920, 0x26, 0x30}, {0x174301109322d676, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023239548}, {{0x174301109322d676, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109322d676, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc050ef2200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b0f03b76, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0160b1400, {0x63c85b8, 0xc062cde450}, 0xc050ef2200, 0xc077741540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0160b1400, {0x63c85b8, 0xc062cde450}, 0xc050ef2200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109322d676, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109322d676, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cde3c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cde3c0}, {0xc0392ed3f8, 0x660089}, 0xc0392ed400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cde3c0}, {0x7fe3b8a58d10}, 0xc050ef2000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cde330}, 0xc050ef2000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}, 0xc06ac12a20, 0xc041139f20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}, 0x3, 0xc01c0bd840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0392ed980}}}, {0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}, 0xc06ac12a20, 0xc01c0bd840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cde330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cde330}, {0x40, 0xc0392edb70}, 0xc0392edab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cde330}, {0x4e9d7a0, 0xc050ef2000}, 0xc06ac12a20, 0xc041139f20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cde330}, 0xc049257320, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a5560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a5560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219114755 [runnable]: syscall.Syscall(0x1, 0x1a, 0xc07da9f000, 0x24d) GOROOT/src/syscall/asm_linux_amd64.s:20 +0x5 syscall.write(0xc0272da480, {0xc07da9f000, 0x4700, 0xc07da9f000}) GOROOT/src/syscall/zsyscall_linux_amd64.go:915 +0x4d syscall.Write(...) GOROOT/src/syscall/syscall_unix.go:214 internal/poll.ignoringEINTRIO(...) GOROOT/src/internal/poll/fd_unix.go:582 internal/poll.(*FD).Write(0xc0272da480, {0xc07da9f000, 0x24d, 0x4800}) GOROOT/src/internal/poll/fd_unix.go:275 +0x36e net.(*netFD).Write(0xc0272da480, {0xc07da9f000, 0x0, 0x0}) GOROOT/src/net/fd_posix.go:74 +0x29 net.(*conn).Write(0xc046cec5a8, {0xc07da9f000, 0x0, 0x0}) GOROOT/src/net/net.go:195 +0x45 crypto/tls.(*Conn).write(0xc01f6ad500, {0xc07da9f000, 0x5, 0x4800}) GOROOT/src/crypto/tls/conn.go:913 +0x108 crypto/tls.(*Conn).writeRecordLocked(0xc01f6ad500, 0x17, {0xc07bffaa00, 0x237, 0x21d3}) GOROOT/src/crypto/tls/conn.go:981 +0x351 crypto/tls.(*Conn).Write(0x475c5a0, {0xc07bffaa00, 0x237, 0x21d3}) GOROOT/src/crypto/tls/conn.go:1152 +0x405 bytes.(*Buffer).WriteTo(0xc0866cb270, {0x7fe422ac65d0, 0xc02ddbede0}) GOROOT/src/bytes/buffer.go:239 +0x83 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).Flush(0xc0866caa80, 0x0) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1593 +0xfa github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*commandResult).Close(0xc0866cb140, {0x63c8510, 0xc063739bc0}, 0x85) github.com/cockroachdb/cockroach/pkg/sql/pgwire/command_result.go:163 +0x339 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc02edab300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2114 +0x1537 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc02edab300, {0x63c8510, 0xc063739bc0}, 0xc056a5fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07ac388a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc063739bc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07ac388a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812531 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074aa90, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04bc39600, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc053e30000, 0xc04bc39600, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc04bc39600, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a526e0, 0xc03ed2d198, 0xc0dca47090, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc0dca471a8, 0x0, 0xc0dca471a8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0dca47040, 0xc005cf6000, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0dca47040, {0xc09092bea0, 0x2b, 0x6c}, {0xc03a9923f0, 0x2b, 0x2b}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0dca45bc0, {0xc03a9923f0, 0x4a2372, 0x2b}, 0x464ed60, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0dca45bc0, {0xc09092bea0, 0x2b, 0x2b}, {0xc03a9923f0, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0dca45900, {0xc03a9923f0, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc07429c008, {{0xc03a9923c0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0936d3000, {{0xc03a9923c0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc049922c00, {0x63c85b8, 0xc01786c600}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01786c600}, {0x64a6560, 0xc0936d3000}, {0xc03a9923c0, 0x32444f12e744d1ca, 0x30}, {0x17430110828833c8, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01786c600}, {0x6466ed8, 0xc07429c000}, {0xc03a9923c0, 0x2a, 0x30}, {0x17430110828833c8, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0bb0a97a0}, {{0x17430110828833c8, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110828833c8, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06bbd5200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a05598c8, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01de0d000, {0x63c85b8, 0xc01786c600}, 0xc06bbd5200, 0xc07a3a7cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01de0d000, {0x63c85b8, 0xc01786c600}, 0xc06bbd5200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110828833c8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110828833c8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01786c420}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01786c420}, {0xc03ed333f8, 0x660089}, 0xc03ed33400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01786c420}, {0x1700898901003d3d}, 0xc06bbd5000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01786c2a0}, 0xc06bbd5000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}, 0xc0ccf7f320, 0xc038160fd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}, 0x3, 0xc049737d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03ed33980}}}, {0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}, 0xc0ccf7f320, 0xc049737d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01786c2a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01786c2a0}, {0x40, 0xc03ed33b70}, 0xc03ed33ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01786c2a0}, {0x4e9d7a0, 0xc06bbd5000}, 0xc0ccf7f320, 0xc038160fd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01786c2a0}, 0xc021af14a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdcea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdcea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219793075 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc033612180}, 0xc0385e5cb0, {0x646cc48, 0xc0336121e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc033612180}, 0xedb7a834e, 0xc058bc5e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc033612180}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc059b060c0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0385e5ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc059b060c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806389 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04da93d70}, 0xc08ba3fc08, 0x1, 0xc006f8b0a8, 0x519bbc, 0xc0402acf60, 0xc0912fa600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04da93d70}, 0x1e0, 0xc006f8b230, 0x402acf00, 0x0, 0x660069, 0xc0402acf60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04da93d70}, 0xc0402acf00, {{{{0x0, 0x0}, {0xc08f5f38c0, 0x101}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04da93d70}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04da93d70}, {0xc0785cad80, {0x174301104c829237, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04da93d70}, 0xc07d3cc5a0, 0xc006f8b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04da93d70}, 0x660089, {0xc0785cad80, {0x174301104c829237, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04da93d70}, 0xc0847b2500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c829237, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c829237, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04da93ce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04da93ce0}, {0xc006f8d3f8, 0x660089}, 0xc006f8d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04da93ce0}, {0x7fe3fb7189e8}, 0xc0847b2300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04da93c50}, 0xc0847b2300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}, 0xc07c5b0a20, 0xc08fc7a228) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}, 0x3, 0xc06e489400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006f8d980}}}, {0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}, 0xc07c5b0a20, 0xc06e489400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04da93c50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04da93c50}, {0x40, 0xc006f8db70}, 0xc006f8dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04da93c50}, {0x4e9d7a0, 0xc0847b2300}, 0xc07c5b0a20, 0xc08fc7a228) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04da93c50}, 0xc052475620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc046333320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc046333320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804621 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc093b418c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802834 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b811800, {0x63c85b8, 0xc018a23380}, 0xc04c13f000, 0xc0ad173220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b811800, {0x63c85b8, 0xc018a23380}, 0xc04c13f000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102561b4db, 0x0, 0x0}, 0xc04827faa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102561b4db, 0x0, 0x0}, 0xc04827faa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc018a232f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018a232f0}, {0xc05ff453f8, 0x660089}, 0xc05ff45400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc018a232f0}, {0x7fe13e89ce98}, 0xc04c13ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018a23200}, 0xc04c13ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}, 0xc0736d85e0, 0xc041fa8018) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}, 0x3, 0xc042d2db80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05ff45980}}}, {0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}, 0xc0736d85e0, 0xc042d2db80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc018a23200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018a23200}, {0x40, 0xc05ff45b70}, 0xc05ff45ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc018a23200}, {0x4e9d7a0, 0xc04c13ee00}, 0xc0736d85e0, 0xc041fa8018) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc018a23200}, 0xc077ef0f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04bf1db00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04bf1db00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807729 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ff75c00, {0x63c85b8, 0xc0380d3cb0}, 0xc0ce8f3d00, 0xc0463c0c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75c00, {0x63c85b8, 0xc0380d3cb0}, 0xc0ce8f3d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011059e37ae0, 0x0, 0x0}, 0xc010406c70, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011059e37ae0, 0x0, 0x0}, 0xc010406c70, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0380d3bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0380d3bf0}, {0xc00e8213f8, 0x660089}, 0xc00e821400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0380d3bf0}, {0x7fe425757b58}, 0xc0ce8f3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0380d3b60}, 0xc0ce8f3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}, 0xc081bc8c80, 0xc09c322720) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}, 0x3, 0xc04a090d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e821980}}}, {0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}, 0xc081bc8c80, 0xc04a090d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0380d3b60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0380d3b60}, {0x40, 0xc00e821b70}, 0xc00e821ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0380d3b60}, {0x4e9d7a0, 0xc0ce8f3b00}, 0xc081bc8c80, 0xc09c322720) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0380d3b60}, 0xc015be0060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc071835c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc071835c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219711400 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe79f48, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0710b0900, 0xc003869000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0710b0900, {0xc003869000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0710b0900, {0xc003869000, 0xc02032a8c0, 0xc02f814b40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0210ac208, {0xc003869000, 0x0, 0x0}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0842aacc0, {0xc003869000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc01dc60018, {0xc003869000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc067be0cf8, {0x631b900, 0xc01dc60018}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc067be0a80, {0x7fe42ab59310, 0xc0842aacc0}, 0x7fe13d3b6600) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc067be0a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc067be0a80, {0xc0a4efa000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0a0b11b60, {0xc0a4efa000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0aa9b7588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0aa9b7588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0aa9b7b60, {0x634aac0, 0xc0aa9b7588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0aa9b7500, 0xc06c818bf0, 0xc07e03942b, 0xc07e03942d, {0x635bec0, 0xc064a14000}, 0xc000d6ae00, 0xc01f59bf68, 0xc07e039429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0aa9b7500, {0x63c85b8, 0xc02705e030}, 0xc06c818be0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc085bcf009, 0x8}}, 0x0, 0xc03e109fb0, 0xc02705e000, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02705e030}, {0x643d810, 0xc0842aacc0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc021623d10}, {0x643d810, 0xc0842aacc0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc021623d10}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791651 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc08230f8c0}, 0xc0387f6cb0, {0x646cc48, 0xc08230f920}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc08230f8c0}, 0xedb7a834e, 0xc0895eb200) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc08230f8c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0772dc2a0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0387f6ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0772dc2a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804932 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ecbf400, {0x63c85b8, 0xc046a5f890}, 0xc0299e28a0, 0x1, 0x0, 0x1, 0xc06f5aa3c0, 0xc058b7c3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046a5f890}, 0x120, 0xc0942dc728, 0x6f5aa360, 0xc03f09ab60, 0x18, 0xc06f5aa3c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ecbf400, {0x63c85b8, 0xc046a5f890}, 0xc06f5aa360, {{{{0x0, 0x0}, {0xc01e9418c0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046a5f890}, 0x6352410, 0x71c3e3c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01eb85e50, {0x63c85b8, 0xc046a5f890}, {0xc06eb5ab40, {0x174301103269927a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01eb85e50, {0x63c85b8, 0xc046a5f890}, 0xc083df4140, 0xc07c43eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046a5f890}, 0x660089, {0xc06eb5ab40, {0x174301103269927a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ed2e800, {0x63c85b8, 0xc046a5f890}, 0xc08dbb3a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103269927a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103269927a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046a5f800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046a5f800}, {0xc0059ec8f0, 0x660089}, 0xc0059ec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046a5f800}, {0xc00102e4e0}, 0xc08dbb3800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046a5f7a0}, 0xc08dbb3800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc077cd1b00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802945 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02c4a8c00, {0x63c85b8, 0xc04aee8600}, 0xc042123e00, 0xc07390dcc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc04aee8600}, 0xc042123e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011028eb35d8, 0x0, 0x0}, 0xc03d5d4690, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011028eb35d8, 0x0, 0x0}, 0xc03d5d4690, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04aee8570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aee8570}, {0xc06da7b3f8, 0x660089}, 0xc06da7b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04aee8570}, {0xc058520780}, 0xc042123c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04aee84e0}, 0xc042123c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}, 0xc024d86400, 0xc04025d290) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}, 0x3, 0xc06e42b200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06da7b980}}}, {0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}, 0xc024d86400, 0xc06e42b200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04aee84e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aee84e0}, {0x40, 0xc06da7bb70}, 0xc06da7bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04aee84e0}, {0x4e9d7a0, 0xc042123c00}, 0xc024d86400, 0xc04025d290) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04aee84e0}, 0xc045f6b740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060c82480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060c82480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803104 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0302e5000, {0x63c85b8, 0xc0390672c0}, 0xc0552aa900, 0xc07ae3b220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0302e5000, {0x63c85b8, 0xc0390672c0}, 0xc0552aa900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102a46b5e1, 0x0, 0x0}, 0xc0631e7400, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102a46b5e1, 0x0, 0x0}, 0xc0631e7400, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039067230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039067230}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039067230}, {0xc00102e4e0}, 0xc0552aa800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0390671a0}, 0xc0552aa800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc0217ac300}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc039066fc0}, {0xc00dfd36b0, 0x24}, 0x46c7a4656, 0xc0385e1d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x4c1af3}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc039066fc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803570 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc079f00a50}, 0xc07228b680, 0x1, 0xc07643d0a8, 0x519bbc, 0xc046f7baa0, 0xc0455c78c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc079f00a50}, 0x1e0, 0xc07643d230, 0x46f7ba40, 0x0, 0x660069, 0xc046f7baa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc079f00a50}, 0xc046f7ba40, {{{{0x0, 0x0}, {0xc03e7b3680, 0x42}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc079f00a50}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc079f00a50}, {0xc03b5cd320, {0x174301102d482bda, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc079f00a50}, 0xc073d9f5e0, 0xc07643d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc079f00a50}, 0x660089, {0xc03b5cd320, {0x174301102d482bda, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc079f00a50}, 0xc078d5e700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d482bda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d482bda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079f00990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079f00990}, {0xc07643f3f8, 0x660089}, 0xc07643f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079f00990}, {0xc074576360}, 0xc078d5e500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079f00900}, 0xc078d5e500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}, 0xc04547cfa0, 0xc047a004f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}, 0x3, 0xc00c0bf700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07643f980}}}, {0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}, 0xc04547cfa0, 0xc00c0bf700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079f00900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079f00900}, {0x40, 0xc07643fb70}, 0xc07643fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079f00900}, {0x4e9d7a0, 0xc078d5e500}, 0xc04547cfa0, 0xc047a004f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079f00900}, 0xc02ae511a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03b5cd200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03b5cd200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801067 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02c4a8c00, {0x63c85b8, 0xc024a034d0}, 0xc04e692400, 0xc06f9661e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc024a034d0}, 0xc04e692400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101bdcad7a, 0x0, 0x0}, 0xc044a06660, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101bdcad7a, 0x0, 0x0}, 0xc044a06660, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024a03320}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024a03320}, {0xc075e732a8, 0x660089}, 0xc075e732b0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024a03320}, {0xc00102e4e0}, 0xc04e692300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024a031d0}, 0xc04e692300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8580, 0xc0743b65a0}, 0x0, 0xc051c64580) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c8580, 0xc0743b65a0}, 0x63c8580) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftLogQueue).process(0xc003629840, {0x63c8580, 0xc0743b65a0}, 0xc02c4a8c00, {0xc0743b65a0, 0x65f500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_log_queue.go:740 +0x87c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica.func1({0x63c8580, 0xc0743b65a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:977 +0x26e github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc024a02ff0}, {0xc0829ce2d0, 0x23}, 0xdf8475800, 0xc0933e3e28) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica(0xc00d506f00, {0x63c85b8, 0xc024a02fc0}, {0x6463f50, 0xc02c4a8c00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:936 +0x3f5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2.1({0x63c85b8, 0xc024a02fc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:855 +0xde github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810217 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01dd1a800, {0x63c85b8, 0xc0501bab40}, 0xc053813b00, 0xc0720b0960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd1a800, {0x63c85b8, 0xc0501bab40}, 0xc053813b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c17d88c, 0x0, 0x0}, 0xc05abdc910, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c17d88c, 0x0, 0x0}, 0xc05abdc910, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0501baab0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0501baab0}, {0xc0179673f8, 0x660089}, 0xc017967400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0501baab0}, {0x0}, 0xc053813900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0501baa20}, 0xc053813900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}, 0xc0701d5a80, 0xc048c9aff0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}, 0x3, 0xc02853bd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017967980}}}, {0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}, 0xc0701d5a80, 0xc02853bd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0501baa20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0501baa20}, {0x40, 0xc017967b70}, 0xc017967ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0501baa20}, {0x4e9d7a0, 0xc053813900}, 0xc0701d5a80, 0xc048c9aff0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0501baa20}, 0xc01e2c93e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00d59db00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00d59db00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794234 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa01e8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc063f92980, 0xc00715d800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc063f92980, {0xc00715d800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc063f92980, {0xc00715d800, 0x735, 0xc0505e8c80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06b067310, {0xc00715d800, 0x7eda79, 0xc00715d805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc05ed5b4a0, {0xc00715d800, 0xc078c1ec60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc041138480, {0xc00715d800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc082a3fe78, {0x631b900, 0xc041138480}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc082a3fc00, {0x7fe42ab59310, 0xc05ed5b4a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc082a3fc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc082a3fc00, {0xc0b9646000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc040a5a7e0, {0xc0b9646000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0126e8088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0126e8088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0126e8660, {0x634aac0, 0xc0126e8088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0126e8000, 0xc017bd5930, 0xc078c1f42b, 0xc078c1f42d, {0x635bec0, 0xc04c36c2d0}, 0xc000d6ae00, 0xc054a7c1e8, 0xc078c1f429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0126e8000, {0x63c85b8, 0xc070be3020}, 0xc017bd5920, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06b7b9009, 0x8}}, 0x0, 0xc070be2fc0, 0xc070be2ff0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc070be3020}, {0x643d810, 0xc05ed5b4a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc033bf71a0}, {0x643d810, 0xc05ed5b4a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc033bf71a0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811921 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ef69540, {0x63c85b8, 0xc05c285ad0}, 0xc083a1c3c0, 0x1, 0xc03ea190a8, 0x519bbc, 0xc0117ad260, 0xc05ba3a840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05c285ad0}, 0x120, 0xc03ea19230, 0x117ad200, 0x0, 0x660069, 0xc0117ad260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ef69540, {0x63c85b8, 0xc05c285ad0}, 0xc0117ad200, {{{{0x0, 0x0}, {0xc074ada6c0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05c285ad0}, 0x6352410, 0x3c90c590) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01ef6ca00, {0x63c85b8, 0xc05c285ad0}, {0xc0117acfc0, {0x174301107f056b0c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01ef6ca00, {0x63c85b8, 0xc05c285ad0}, 0xc0688e8be0, 0xc03ea199f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05c285ad0}, 0x660089, {0xc0117acfc0, {0x174301107f056b0c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f011c00, {0x63c85b8, 0xc05c285ad0}, 0xc02664f500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f056b0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f056b0c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05c2859e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c2859e0}, {0xc03ea1b3f8, 0x660089}, 0xc03ea1b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05c2859e0}, {0x10420000d005}, 0xc02664f300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05c285920}, 0xc02664f300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}, 0xc03d9ad6e0, 0xc023415c68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}, 0x3, 0xc048fdec00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03ea1b980}}}, {0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}, 0xc03d9ad6e0, 0xc048fdec00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05c285920}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c285920}, {0x40, 0xc03ea1bb70}, 0xc03ea1bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05c285920}, {0x4e9d7a0, 0xc02664f300}, 0xc03d9ad6e0, 0xc023415c68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05c285920}, 0xc092bf2720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0117ac480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0117ac480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807308 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02a288800, {0x63c85b8, 0xc049a3f020}, 0xc08d7f3e00, 0xc036f7dc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a288800, {0x63c85b8, 0xc049a3f020}, 0xc08d7f3e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110570022d4, 0x0, 0x0}, 0xc08ee82da0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110570022d4, 0x0, 0x0}, 0xc08ee82da0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc049a3ef90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049a3ef90}, {0xc0054f93f8, 0x660089}, 0xc0054f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc049a3ef90}, {0x40df00bf96283369}, 0xc08d7f3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc049a3ef00}, 0xc08d7f3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}, 0xc062506a00, 0xc0a55fedb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}, 0x3, 0xc01c2341c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0054f9980}}}, {0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}, 0xc062506a00, 0xc01c2341c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc049a3ef00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049a3ef00}, {0x40, 0xc0054f9b70}, 0xc0054f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc049a3ef00}, {0x4e9d7a0, 0xc08d7f3c00}, 0xc062506a00, 0xc0a55fedb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc049a3ef00}, 0xc0474f90e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07b8f9560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07b8f9560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794830 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc00e8a7050) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc00e8a70b8, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc072b645f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0be91bc00, {0xc016d10e80, 0x2b, 0x74}, {0xc0240fc8c0, 0x2b, 0x6e}, 0x1, 0x45) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc072b64488, {0xc016d10e80, 0x1, 0x6e}, {0xc0240fc8c0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc072b64488, {0xc016d10e80, 0x2b, 0x74}, {0xc0240fc8c0, 0x2b, 0x6e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc072b63bc0, {0xc0240fc8c0, 0x6e, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc072b63bc0, {0xc016d10e80, 0x2b, 0xc0240fc8c0}, {0xc0240fc8c0, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc072b63900, {0xc0240fc8c0, 0x2b, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0532e5b88, {{0xc08d3bdbf0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0b87c6d00, {{0xc08d3bdbf0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0497f6400, {0x63c85b8, 0xc02316b920}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02316b920}, {0x64a6560, 0xc0b87c6d00}, {0xc08d3bdbf0, 0x0, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02316b920}, {0x6466ed8, 0xc0532e5b80}, {0xc08d3bdbf0, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0672e9d58}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc047f78e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b11000, {0x63c85b8, 0xc02316b920}, 0xc047f78e00, 0xc055e5fc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b11000, {0x63c85b8, 0xc02316b920}, 0xc047f78e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02316b890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02316b890}, {0xc00e8ad3f8, 0x660089}, 0xc00e8ad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02316b890}, {0xc06f830f40}, 0xc047f78b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02316b740}, 0xc047f78b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}, 0xc05b26be20, 0xc06cff53e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}, 0x3, 0xc029e73f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e8ad980}}}, {0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}, 0xc05b26be20, 0xc029e73f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02316b740}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02316b740}, {0x40, 0xc00e8adb70}, 0xc00e8adab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02316b740}, {0x4e9d7a0, 0xc047f78b00}, 0xc05b26be20, 0xc06cff53e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02316b740}, 0xc022b67680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc014a3b560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc014a3b560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811298 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024f41400, {0x63c85b8, 0xc0350e38c0}, 0xc06df83b00, 0xc070e27860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024f41400, {0x63c85b8, 0xc0350e38c0}, 0xc06df83b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110744f4d41, 0x0, 0x0}, 0xc0449c88e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110744f4d41, 0x0, 0x0}, 0xc0449c88e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0350e3830}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0350e3830}, {0xc01fb1b3f8, 0x660089}, 0xc01fb1b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0350e3830}, {0x7fe13e8e5c30}, 0xc06df83900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0350e37a0}, 0xc06df83900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}, 0xc00cd96800, 0xc04916db30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}, 0x3, 0xc042b0a540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fb1b980}}}, {0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}, 0xc00cd96800, 0xc042b0a540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0350e37a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0350e37a0}, {0x40, 0xc01fb1bb70}, 0xc01fb1bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0350e37a0}, {0x4e9d7a0, 0xc06df83900}, 0xc00cd96800, 0xc04916db30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0350e37a0}, 0xc06026d980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc00c4bf8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc00c4bf8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802685 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc045df6de0}, 0xc0b9ef0400, 0xc05c4481e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc045df6de0}, 0xc0b9ef0400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024800fcf, 0x0, 0x0}, 0xc03c6a6c30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024800fcf, 0x0, 0x0}, 0xc03c6a6c30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045df6d50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045df6d50}, {0xc05c84d3f8, 0x660089}, 0xc05c84d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045df6d50}, {0xc0015221e0}, 0xc0b9ef0100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045df6cc0}, 0xc0b9ef0100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}, 0xc03d253640, 0xc068a75f68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}, 0x3, 0xc020497600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05c84d980}}}, {0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}, 0xc03d253640, 0xc020497600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045df6cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045df6cc0}, {0x40, 0xc05c84db70}, 0xc05c84dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045df6cc0}, {0x4e9d7a0, 0xc0b9ef0100}, 0xc03d253640, 0xc068a75f68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045df6cc0}, 0xc021d7fe60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799947 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x68) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x100, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0933976f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc010984700, {0xc057650900, 0x32, 0x90}, {0xc02ea0b100, 0x32, 0x71}, 0x1, 0x76) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc093397588, {0xc057650900, 0x1, 0x71}, {0xc02ea0b100, 0x2b0, 0xc095648790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093397588, {0xc057650900, 0x32, 0x90}, {0xc02ea0b100, 0x32, 0x71}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc093396cc0, {0xc02ea0b100, 0x71, 0x90}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc093396cc0, {0xc057650900, 0x71, 0xc02ea0b100}, {0xc02ea0b100, 0xc055c7b0e0, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093396a00, {0xc02ea0b100, 0x32, 0x71}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04a2acda8, {{0xc055c7b0e0, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04e790300, {{0xc08dc6b440, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05195cc00, {0x63c85b8, 0xc02e180ba0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02e180ba0}, {0x64a6560, 0xc04e790300}, {0xc08dc6b440, 0x28cdb, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02e180ba0}, {0x6466ed8, 0xc04a2acb00}, {0xc08dc6b440, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc013fc1ea8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc017fce100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209eec00, {0x63c85b8, 0xc02e180ba0}, 0xc017fce100, 0xc067978aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209eec00, {0x63c85b8, 0xc02e180ba0}, 0xc017fce100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02e180b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02e180b10}, {0xc0076ca8f0, 0x660089}, 0xc0076ca8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02e180b10}, {0x24}, 0xc017fce000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02e180a20}, 0xc017fce000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc024122420}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808038 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc0709e3aa0}, 0xc016a613b0, 0x1, 0xc00f5a90a8, 0x519bbc, 0xc0749c6a80, 0xc05baa1800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0709e3aa0}, 0x120, 0xc00f5a9230, 0x749c6a20, 0x0, 0x660069, 0xc0749c6a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc0709e3aa0}, 0xc0749c6a20, {{{{0x0, 0x0}, {0xc05c03a900, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0709e3aa0}, 0x6352410, 0x13b805a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc0709e3aa0}, {0xc0749c6900, {0x174301105988909c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc0709e3aa0}, 0xc01d4fe1e0, 0xc00f5a99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0709e3aa0}, 0x660089, {0xc0749c6900, {0x174301105988909c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0709e3aa0}, 0xc0ae7d6900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105988909c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105988909c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0709e3a10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0709e3a10}, {0xc00f5ab3f8, 0x660089}, 0xc00f5ab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0709e3a10}, {0x7fe3e0ed55c0}, 0xc0ae7d6700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0709e3980}, 0xc0ae7d6700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}, 0xc006149a20, 0xc027349cf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}, 0x3, 0xc012341dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f5ab980}}}, {0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}, 0xc006149a20, 0xc012341dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0709e3980}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0709e3980}, {0x40, 0xc00f5abb70}, 0xc00f5abab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0709e3980}, {0x4e9d7a0, 0xc0ae7d6700}, 0xc006149a20, 0xc027349cf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0709e3980}, 0xc0190c78c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0749c67e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0749c67e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812657 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219792657 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc058eb0360}, 0xc0198a0cb0, {0x646cc48, 0xc058eb03c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc058eb0360}, 0xedb7a834e, 0xc06e0ed560) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc058eb0360}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc047c5c570}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0198a0ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc047c5c570}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812797 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803361 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc025584210}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219797886 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc01e577f20}, 0xc04da00cb0, {0x646cc48, 0xc01e577f80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc01e577f20}, 0xedb7a834e, 0xc0437f3e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc01e577f20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02ea7dad0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc04da00ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc02ea7dad0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803725 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc04c6f74a0}, 0xc046e05a88, 0x1, 0xc066d5b0a8, 0x519bbc, 0xc0627a5ec0, 0xc05762dda0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04c6f74a0}, 0x120, 0xc066d5b230, 0x627a5e60, 0x0, 0x660069, 0xc0627a5ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc04c6f74a0}, 0xc0627a5e60, {{{{0x0, 0x0}, {0xc048501d40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04c6f74a0}, 0x6352410, 0x33b751b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc04c6f74a0}, {0xc0627a5d40, {0x17430110356c23ac, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc04c6f74a0}, 0xc071b18320, 0xc066d5b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c6f74a0}, 0x660089, {0xc0627a5d40, {0x17430110356c23ac, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc04c6f74a0}, 0xc04cffbf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110356c23ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110356c23ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c6f7410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c6f7410}, {0xc066d5d3f8, 0x660089}, 0xc066d5d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c6f7410}, {0xc049977630}, 0xc04cffbd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c6f7380}, 0xc04cffbd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}, 0xc052b6a400, 0xc025fef530) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}, 0x3, 0xc021f47200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc066d5d980}}}, {0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}, 0xc052b6a400, 0xc021f47200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04c6f7380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c6f7380}, {0x40, 0xc066d5db70}, 0xc066d5dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04c6f7380}, {0x4e9d7a0, 0xc04cffbd00}, 0xc052b6a400, 0xc025fef530) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04c6f7380}, 0xc047cf44e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0627a5c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0627a5c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812755 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809587 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030b32c00, {0x63c85b8, 0xc0682a9890}, 0xc058b31900, 0xc0123ead20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030b32c00, {0x63c85b8, 0xc0682a9890}, 0xc058b31900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011064ff58cd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011064ff58cd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0682a9800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0682a9800}, {0xc026baf3f8, 0x660089}, 0xc026baf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0682a9800}, {0x33}, 0xc058b31700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0682a9770}, 0xc058b31700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}, 0xc0472a2f40, 0xc0337f8d68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}, 0x3, 0xc01b65d4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026baf980}}}, {0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}, 0xc0472a2f40, 0xc01b65d4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0682a9770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0682a9770}, {0x40, 0xc026bafb70}, 0xc026bafab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0682a9770}, {0x4e9d7a0, 0xc058b31700}, 0xc0472a2f40, 0xc0337f8d68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0682a9770}, 0xc043ace360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07c3be5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07c3be5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219632972 [runnable]: internal/poll.runtime_pollWait(0x7fdbafe7a218, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0789ec800, 0xc00d5e8000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0789ec800, {0xc00d5e8000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0789ec800, {0xc00d5e8000, 0x735, 0xc06bd36a00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc030992da8, {0xc00d5e8000, 0x7eda79, 0xc00d5e8005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc08bb201e0, {0xc00d5e8000, 0x3, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc018c97fb0, {0xc00d5e8000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0605365f8, {0x631b900, 0xc018c97fb0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc060536380, {0x7fe42ab59310, 0xc08bb201e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc060536380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc060536380, {0xc036018000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc078656a08, {0xc036018000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0cbf98088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0cbf98088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0cbf98660, {0x634aac0, 0xc0cbf98088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0cbf98000, 0xc089e69fe0, 0xc08969142b, 0xc08969142d, {0x635bec0, 0xc026a49c20}, 0xc000d6ae00, 0xc059870c28, 0xc089691429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0cbf98000, {0x63c85b8, 0xc03b704b40}, 0xc089e69fd0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc051f00021, 0x8}}, 0x0, 0xc03b704ae0, 0xc03b704b10, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03b704b40}, {0x643d810, 0xc08bb201e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0429346f0}, {0x643d810, 0xc08bb201e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0429346f0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803239 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc02fa7dc50}, 0xc015fb3ec0, 0x1, 0xc074fc50a8, 0x519bbc, 0xc0347e23c0, 0xc0347a83c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02fa7dc50}, 0x120, 0xc074fc5230, 0x347e2360, 0x0, 0x660069, 0xc0347e23c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc02fa7dc50}, 0xc0347e2360, {{{{0x0, 0x0}, {0xc079ab0240, 0x15}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02fa7dc50}, 0x6352410, 0x473dd168) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc02fa7dc50}, {0xc0347e2240, {0x174301102b2a2ba3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc02fa7dc50}, 0xc01cc9aaa0, 0xc074fc59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02fa7dc50}, 0x660089, {0xc0347e2240, {0x174301102b2a2ba3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc02fa7dc50}, 0xc062c14f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b2a2ba3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b2a2ba3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02fa7db00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fa7db00}, {0xc074fc73f8, 0x660089}, 0xc074fc7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02fa7db00}, {0xc0070964c0}, 0xc062c14d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02fa7da70}, 0xc062c14d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}, 0xc04622e060, 0xc065e046d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}, 0x3, 0xc062b436c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc074fc7980}}}, {0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}, 0xc04622e060, 0xc062b436c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02fa7da70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fa7da70}, {0x40, 0xc074fc7b70}, 0xc074fc7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02fa7da70}, {0x4e9d7a0, 0xc062c14d00}, 0xc04622e060, 0xc065e046d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02fa7da70}, 0xc01c8bfa40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03593c240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03593c240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804872 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021cd6000, {0x63c85b8, 0xc041dc03c0}, 0xc0878b0a00, 0xc0d0452140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd6000, {0x63c85b8, 0xc041dc03c0}, 0xc0878b0a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b05a390, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b05a390, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041dc0330}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041dc0330}, {0xc05b49b3f8, 0x660089}, 0xc05b49b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041dc0330}, {0x7fe3f1e6c558}, 0xc0878b0800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041dc02a0}, 0xc0878b0800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}, 0xc0235c4c80, 0xc031857bf0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}, 0x3, 0xc0499eccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05b49b980}}}, {0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}, 0xc0235c4c80, 0xc0499eccc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041dc02a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041dc02a0}, {0x40, 0xc05b49bb70}, 0xc05b49bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}, 0x203012, 0x203012) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041dc02a0}, {0x4e9d7a0, 0xc0878b0800}, 0xc0235c4c80, 0xc031857bf0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041dc02a0}, 0xc04816da40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06ee170e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06ee170e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219748353 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3df7d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc059337500, 0xc053aef000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc059337500, {0xc053aef000, 0x2b87, 0x2b87}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc059337500, {0xc053aef000, 0x2b87, 0xc074c6f540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0476a1038, {0xc053aef000, 0x7eda79, 0xc053aef005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02bd765a0, {0xc053aef000, 0xc00b2c2c78, 0x2b87}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03ed24b70, {0xc053aef000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06d52f078, {0x631b900, 0xc03ed24b70}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06d52ee00, {0x7fe42ab59310, 0xc02bd765a0}, 0x2b87) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06d52ee00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06d52ee00, {0xc0b03b6000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc054e74a08, {0xc0b03b6000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc048426b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc048426b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0484270e0, {0x634aac0, 0xc048426b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc048426a80, 0xc056311970, 0xc065c4742b, 0xc065c4742d, {0x635bec0, 0xc06cb4e090}, 0xc000d6ae00, 0xc01c15f414, 0xc065c47429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc048426a80, {0x63c85b8, 0xc08591cd20}, 0xc056311960, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09e4bd037, 0x8}}, 0x0, 0xc08591ccc0, 0xc08591ccf0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc08591cd20}, {0x643d810, 0xc02bd765a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03bf25c80}, {0x643d810, 0xc02bd765a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03bf25c80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806636 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0218b2800, {0x63c85b8, 0xc0909a96b0}, 0xc097d97700, 0xc0632612c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b2800, {0x63c85b8, 0xc0909a96b0}, 0xc097d97700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104efe0834, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104efe0834, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0909a9620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0909a9620}, {0xc003ec13f8, 0x660089}, 0xc003ec1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0909a9620}, {0x7fe34e262170}, 0xc097d97500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0909a9590}, 0xc097d97500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}, 0xc08e327fe0, 0xc083a1c690) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}, 0x3, 0xc08e4dbd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003ec1980}}}, {0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}, 0xc08e327fe0, 0xc08e4dbd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0909a9590}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0909a9590}, {0x40, 0xc003ec1b70}, 0xc003ec1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0909a9590}, {0x4e9d7a0, 0xc097d97500}, 0xc08e327fe0, 0xc083a1c690) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0909a9590}, 0xc06d8e21e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc008399200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc008399200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812227 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030065c00, {0x63c85b8, 0xc0111c6390}, 0xc077ba0e00, 0xc05f6945a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc0111c6390}, 0xc077ba0e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e4e9dda, 0x0, 0x0}, 0xc04f025700, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e4e9dda, 0x0, 0x0}, 0xc04f025700, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0111c62a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0111c62a0}, {0xc076115a08, 0x660089}, 0xc076115a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0111c62a0}, {0xc00102e4e0}, 0xc077ba0d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0111c60f0}, 0xc077ba0d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc0111c6060}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0111c6060}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802840 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0304b6400, {0x63c85b8, 0xc02a290f30}, 0xc0a9a78d00, 0xc0cef70c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0304b6400, {0x63c85b8, 0xc02a290f30}, 0xc0a9a78d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110270f416b, 0x0, 0x0}, 0xc05c84ec50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110270f416b, 0x0, 0x0}, 0xc05c84ec50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02a290ea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02a290ea0}, {0xc04c2ef918, 0x660089}, 0xc04c2ef920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02a290ea0}, {0xc00102e4e0}, 0xc0a9a78c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02a290e40}, 0xc0a9a78c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc017711680}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02a290d80}, {0xc00dfd36b0, 0x24}, 0x46c7bf774, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc03eaaea20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc02a290d80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219772200 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa0788, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05a3c2600, 0xc00efc9800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05a3c2600, {0xc00efc9800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05a3c2600, {0xc00efc9800, 0x735, 0xc0d0ae1400}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0351a8b30, {0xc00efc9800, 0x40, 0x3f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03e59b0e0, {0xc00efc9800, 0xc0824a6c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc041138420, {0xc00efc9800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0ab199e78, {0x631b900, 0xc041138420}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0ab199c00, {0x7fe42ab59310, 0xc03e59b0e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0ab199c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0ab199c00, {0xc036d74000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04e2ce180, {0xc036d74000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc046b7f588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc046b7f588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc046b7fb60, {0x634aac0, 0xc046b7f588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc046b7f500, 0xc09b369160, 0xc089aed42b, 0xc089aed42d, {0x635bec0, 0xc06d4f7b90}, 0xc000d6ae00, 0xc04ee9f378, 0xc089aed429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc046b7f500, {0x63c85b8, 0xc0b938c2d0}, 0xc09b369150, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc07cd4e03e, 0x8}}, 0x0, 0xc0b938c270, 0xc0b938c2a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0b938c2d0}, {0x643d810, 0xc03e59b0e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc059dcae10}, {0x643d810, 0xc03e59b0e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc059dcae10}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802148 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f22b2c0, {0x63c85b8, 0xc064b06900}, 0xc028ace600, 0x1, 0xc01a3d30a8, 0x519bbc, 0xc09efbeba0, 0xc03db33260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc064b06900}, 0x120, 0xc01a3d3230, 0x9efbeb40, 0x0, 0x660069, 0xc09efbeba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f22b2c0, {0x63c85b8, 0xc064b06900}, 0xc09efbeb40, {{{{0x0, 0x0}, {0xc08e4ef200, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc064b06900}, 0x6352410, 0x351a9940) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f219ae0, {0x63c85b8, 0xc064b06900}, {0xc09efbea20, {0x17430110173aa8b8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f219ae0, {0x63c85b8, 0xc064b06900}, 0xc0403966e0, 0xc01a3d39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc064b06900}, 0x660089, {0xc09efbea20, {0x17430110173aa8b8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02072e800, {0x63c85b8, 0xc064b06900}, 0xc06d00e500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110173aa8b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110173aa8b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064b06870}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064b06870}, {0xc01a3d53f8, 0x660089}, 0xc01a3d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064b06870}, {0x7fe3c15e3970}, 0xc06d00e300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064b06780}, 0xc06d00e300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}, 0xc04ecdac80, 0xc0302f58f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}, 0x3, 0xc06e1c6b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a3d5980}}}, {0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}, 0xc04ecdac80, 0xc06e1c6b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc064b06780}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064b06780}, {0x40, 0xc01a3d5b70}, 0xc01a3d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc064b06780}, {0x4e9d7a0, 0xc06d00e300}, 0xc04ecdac80, 0xc0302f58f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc064b06780}, 0xc04ad08780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0221b4480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0221b4480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801923 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc025040ea0}, 0xc045323b60, 0x1, 0xc04c15d0a8, 0x519bbc, 0xc036b63500, 0xc036759500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc025040ea0}, 0x1e0, 0xc04c15d230, 0x36b634a0, 0x0, 0x660069, 0xc036b63500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc025040ea0}, 0xc036b634a0, {{{{0x0, 0x0}, {0xc03598d440, 0x48}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc025040ea0}, 0x63521a0, 0x65bedd70) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc025040ea0}, {0xc055f36d80, {0x174301101e73bb1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc025040ea0}, 0xc03884f040, 0xc04c15d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc025040ea0}, 0x660089, {0xc055f36d80, {0x174301101e73bb1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc025040ea0}, 0xc07035eb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101e73bb1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101e73bb1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc025040de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc025040de0}, {0xc04c15f3f8, 0x660089}, 0xc04c15f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc025040de0}, {0xc0015221e0}, 0xc07035e900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc025040d50}, 0xc07035e900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}, 0xc0553b7080, 0xc02cf0b9c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}, 0x3, 0xc0d2e65d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04c15f980}}}, {0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}, 0xc0553b7080, 0xc0d2e65d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc025040d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc025040d50}, {0x40, 0xc04c15fb70}, 0xc04c15fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc025040d50}, {0x4e9d7a0, 0xc07035e900}, 0xc0553b7080, 0xc02cf0b9c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc025040d50}, 0xc04f505f20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc055f36900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc055f36900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812052 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc07cd02f00}, 0xc016f61110, 0x1, 0x7080181a2ee8fd, 0x37e11d600, 0xc00d352f00, 0xc04f620f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07cd02f00}, 0x120, 0xc0259e1230, 0xd352ea0, 0x0, 0x660069, 0xc00d352f00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc07cd02f00}, 0xc00d352ea0, {{{{0x0, 0x0}, {0xc0a432cd80, 0x11b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07cd02f00}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc07cd02f00}, {0x0, {0x174301107d82c898, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc07cd02f00}, 0xc07ae3b900, 0xc0259e19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07cd02f00}, 0x660089, {0x0, {0x174301107d82c898, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc07cd02f00}, 0xc082cfc000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107d82c898, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107d82c898, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301107d82c898, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07cd02e70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07cd02e70}, {0xc0259e33f8, 0x660089}, 0xc0259e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07cd02e70}, {0xc0015221e0}, 0xc045013e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07cd02de0}, 0xc045013e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}, 0xc05640f760, 0xc016f610e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}, 0x3, 0xc04708f880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0259e3980}}}, {0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}, 0xc05640f760, 0xc04708f880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07cd02de0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07cd02de0}, {0x40, 0xc0259e3b70}, 0xc0259e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07cd02de0}, {0x4e9d7a0, 0xc045013e00}, 0xc05640f760, 0xc016f610e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07cd02de0}, 0xc021943140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc00d352d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc00d352d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800624 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b9bd400, {0x63c85b8, 0xc02f83df50}, 0xc03454a500, 0xc0710cad20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9bd400, {0x63c85b8, 0xc02f83df50}, 0xc03454a500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110165967e6, 0x0, 0x0}, 0xc09d032db0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110165967e6, 0x0, 0x0}, 0xc09d032db0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f83de30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f83de30}, {0xc03af113f8, 0x660089}, 0xc03af11400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f83de30}, {0xc0015221e0}, 0xc03454a300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f83dc20}, 0xc03454a300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}, 0xc006289520, 0xc0359c2900) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}, 0x3, 0xc029913780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03af11980}}}, {0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}, 0xc006289520, 0xc029913780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f83dc20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f83dc20}, {0x40, 0xc03af11b70}, 0xc03af11ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f83dc20}, {0x4e9d7a0, 0xc03454a300}, 0xc006289520, 0xc0359c2900) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f83dc20}, 0xc0c93d4300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05c253440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05c253440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 207016951 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b2c70, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc062b53580, 0xc0798a1000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc062b53580, {0xc0798a1000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc062b53580, {0xc0798a1000, 0x735, 0xc058694a00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01b3098d8, {0xc0798a1000, 0x7eda79, 0xc0798a1005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc043f845a0, {0xc0798a1000, 0x5, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc072f96cd8, {0xc0798a1000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06eda6cf8, {0x631b900, 0xc072f96cd8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06eda6a80, {0x7fe42ab59310, 0xc043f845a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06eda6a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06eda6a80, {0xc084606000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0269d76e0, {0xc084606000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc084603588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc084603588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc084603b60, {0x634aac0, 0xc084603588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc084603500, 0xc034d91830, 0xc07cd9342b, 0xc07cd9342d, {0x635bec0, 0xc063238c60}, 0xc000d6ae00, 0xc040b16f08, 0xc07cd93429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc084603500, {0x63c85b8, 0xc0118b9aa0}, 0xc034d91820, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc084604087, 0x8}}, 0x0, 0xc0118b9a40, 0xc0118b9a70, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0118b9aa0}, {0x643d810, 0xc043f845a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06c2c2390}, {0x643d810, 0xc043f845a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06c2c2390}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807709 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc06f87b230}, 0xc04598ca00, 0xc049e9b2c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc06f87b230}, 0xc04598ca00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105914effa, 0x0, 0x0}, 0xc092128e30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105914effa, 0x0, 0x0}, 0xc092128e30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f87b170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f87b170}, {0xc0078393f8, 0x660089}, 0xc007839400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f87b170}, {0x7fe425671dc8}, 0xc09949bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f87b0e0}, 0xc09949bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}, 0xc084712a20, 0xc037aeb908) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}, 0x3, 0xc0156dca00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007839980}}}, {0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}, 0xc084712a20, 0xc0156dca00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f87b0e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f87b0e0}, {0x40, 0xc007839b70}, 0xc007839ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f87b0e0}, {0x4e9d7a0, 0xc09949bf00}, 0xc084712a20, 0xc037aeb908) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f87b0e0}, 0xc033f49800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc022e51200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc022e51200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794579 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc032b4d320}, 0xc007b57cb0, {0x646cc48, 0xc032b4d380}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc032b4d320}, 0xedb7a834e, 0xc0401177a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc032b4d320}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07caac9f0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc007b57ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc07caac9f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807447 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0218b1000, {0x63c85b8, 0xc03d4ac7e0}, 0xc08fa63f00, 0xc039b4b220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b1000, {0x63c85b8, 0xc03d4ac7e0}, 0xc08fa63f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057604aaa, 0x0, 0x0}, 0xc09069ec30, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057604aaa, 0x0, 0x0}, 0xc09069ec30, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03d4ac750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d4ac750}, {0xc07b0d5918, 0x660089}, 0xc07b0d5920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03d4ac750}, {0xc00102e4e0}, 0xc08fa63e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03d4ac6f0}, 0xc08fa63e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc021fe3980}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03d4ac660}, {0xc00dfd36b0, 0x24}, 0x46c7b8315, 0xc0918d7d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc03d4ac660}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 216623206 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1bc0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04fb3da80, 0xc01369e800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04fb3da80, {0xc01369e800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04fb3da80, {0xc01369e800, 0x735, 0xc0562b77c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03c90cf78, {0xc01369e800, 0x80, 0x7f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04db9dd40, {0xc01369e800, 0xc07b418c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a90018, {0xc01369e800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04942a5f8, {0x631b900, 0xc043a90018}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04942a380, {0x7fe42ab59310, 0xc04db9dd40}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04942a380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04942a380, {0xc090dfd000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0565740a8, {0xc090dfd000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05460ab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05460ab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05460b0e0, {0x634aac0, 0xc05460ab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05460aa80, 0xc02f22e380, 0xc0588f742b, 0xc0588f742d, {0x635bec0, 0xc00107fa70}, 0xc000d6ae00, 0xc03eaeefb8, 0xc0588f7429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05460aa80, {0x63c85b8, 0xc082192000}, 0xc02f22e360, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06e36a025, 0x8}}, 0x0, 0xc024889f50, 0xc024889fb0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc082192000}, {0x643d810, 0xc04db9dd40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04cd0be30}, {0x643d810, 0xc04db9dd40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04cd0be30}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812305 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc02895b290}, 0xc024451a40, 0x1, 0xc0340410a8, 0x519bbc, 0xc077bd3260, 0xc007a6e600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02895b290}, 0x120, 0xc034041230, 0x77bd3200, 0x0, 0x660069, 0xc077bd3260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc02895b290}, 0xc077bd3200, {{{{0x0, 0x0}, {0xc03cc18fc0, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02895b290}, 0x6352410, 0x6b067020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc02895b290}, {0xc077bd30e0, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc02895b290}, 0xc0846f4c80, 0xc0340419f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02895b290}, 0x660089, {0xc077bd30e0, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc02895b290}, 0xc02751d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02895b200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02895b200}, {0xc0340433f8, 0x660089}, 0xc034043400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02895b200}, {0x19}, 0xc02751d000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02895b170}, 0xc02751d000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}, 0xc0312086a0, 0xc076cdda58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}, 0x3, 0xc045b50bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034043980}}}, {0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}, 0xc0312086a0, 0xc045b50bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02895b170}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02895b170}, {0x40, 0xc034043b70}, 0xc034043ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02895b170}, {0x4e9d7a0, 0xc02751d000}, 0xc0312086a0, 0xc076cdda58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02895b170}, 0xc042686060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc077bd2fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc077bd2fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808133 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc07868a300}, 0xc064cb83f0, 0x1, 0x0, 0x1, 0xc07571eba0, 0xc08d3a6180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07868a300}, 0x120, 0xc00a946728, 0x7571eb40, 0x0, 0x660069, 0xc07571eba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc07868a300}, 0xc07571eb40, {{{{0x0, 0x0}, {0xc058a9a240, 0x29}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07868a300}, 0x6352410, 0x13b806c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc07868a300}, {0xc07571e5a0, {0x1743011057a24815, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc07868a300}, 0xc018f33b80, 0xc076022ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07868a300}, 0x660089, {0xc07571e5a0, {0x1743011057a24815, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc07868a300}, 0xc097ec5b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07868a270}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07868a270}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07868a270}, {0xc00102e4e0}, 0xc097ec5a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07868a210}, 0xc097ec5a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04bcf5d10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805196 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc0688d6f30}, 0xc0418548b8, 0x1, 0xc08d2a90a8, 0x519bbc, 0xc057f01320, 0xc0577cfc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0688d6f30}, 0x1e0, 0xc08d2a9230, 0x57f012c0, 0xc05b97f860, 0x18, 0xc057f01320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc0688d6f30}, 0xc057f012c0, {{{{0x0, 0x0}, {0xc057fd3200, 0x43}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0688d6f30}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc0688d6f30}, {0xc05920f0e0, {0x174301103ff06ae1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc0688d6f30}, 0xc04858a6e0, 0xc08d2a99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0688d6f30}, 0x660089, {0xc05920f0e0, {0x174301103ff06ae1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0688d6f30}, 0xc05897c300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103ff06ae1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103ff06ae1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0688d6ea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0688d6ea0}, {0xc08d2ab3f8, 0x660089}, 0xc08d2ab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0688d6ea0}, {0x4d939c}, 0xc05897c100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0688d6e10}, 0xc05897c100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}, 0xc00af90860, 0xc03f477fc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}, 0x3, 0xc01aae5b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc08d2ab980}}}, {0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}, 0xc00af90860, 0xc01aae5b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0688d6e10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0688d6e10}, {0x40, 0xc08d2abb70}, 0xc08d2abab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}, 0x203006, 0x203006) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0688d6e10}, {0x4e9d7a0, 0xc05897c100}, 0xc00af90860, 0xc03f477fc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0688d6e10}, 0xc0147a95c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05920efc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05920efc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808329 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc07041c300}, 0xc073129400, 0xc022979900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc07041c300}, 0xc073129400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105cdd747e, 0x0, 0x0}, 0xc0180e7860, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105cdd747e, 0x0, 0x0}, 0xc0180e7860, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07041c270}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07041c270}, {0xc005d713f8, 0x660089}, 0xc005d71400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07041c270}, {0xc06f053f80}, 0xc073129200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07041c1e0}, 0xc073129200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}, 0xc037afc400, 0xc038670480) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}, 0x3, 0xc06bb20dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005d71980}}}, {0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}, 0xc037afc400, 0xc06bb20dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07041c1e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07041c1e0}, {0x40, 0xc005d71b70}, 0xc005d71ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07041c1e0}, {0x4e9d7a0, 0xc073129200}, 0xc037afc400, 0xc038670480) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07041c1e0}, 0xc01cc10f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05bc70480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05bc70480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810774 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc025061c00, {0x63c85b8, 0xc015a98d20}, 0xc067eae500, 0xc02b214000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025061c00, {0x63c85b8, 0xc015a98d20}, 0xc067eae500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f7721df, 0x0, 0x0}, 0xc096c52900, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f7721df, 0x0, 0x0}, 0xc096c52900, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc015a98630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015a98630}, {0xc01baf73f8, 0x660089}, 0xc01baf7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc015a98630}, {0x0}, 0xc067eae300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc015a98510}, 0xc067eae300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}, 0xc03e2f55c0, 0xc03bfb38a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}, 0x3, 0xc064fc44c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01baf7980}}}, {0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}, 0xc03e2f55c0, 0xc064fc44c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc015a98510}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015a98510}, {0x40, 0xc01baf7b70}, 0xc01baf7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc015a98510}, {0x4e9d7a0, 0xc067eae300}, 0xc03e2f55c0, 0xc03bfb38a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc015a98510}, 0xc027985b00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc020657e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc020657e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809202 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f5377c0, {0x63c85b8, 0xc040835050}, 0xc02b787fb0, 0x1, 0xc019c130a8, 0x519bbc, 0xc04b16bf60, 0xc072ce13e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc044134480, {0x63c85b8, 0xc040835050}, 0xc04b16bf60, 0xc019c131a8, 0x31, 0x40, 0x0, 0xc04b16bf60, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f5377c0, {0x63c85b8, 0xc040835050}, 0xc0155dc720, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04cbeb320, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc040835050}, 0x63521a0, 0x46e76de8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f552640, {0x63c85b8, 0xc040835050}, {0xc08c0c9b00, {0x1743011002687859, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f552640, {0x63c85b8, 0xc040835050}, 0xc07076a820, 0xc019c139f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc040835050}, 0x660089, {0xc08c0c9b00, {0x1743011002687859, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ffa0400, {0x63c85b8, 0xc040835050}, 0xc0af0c0f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011002687859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011002687859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc040834fc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc040834fc0}, {0xc019c153f8, 0x660089}, 0xc019c15400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc040834fc0}, {0x0}, 0xc0af0c0c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc040834f00}, 0xc0af0c0c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}, 0xc01d037620, 0xc03748cd68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}, 0x3, 0xc067935880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc019c15980}}}, {0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}, 0xc01d037620, 0xc067935880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc040834f00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc040834f00}, {0x40, 0xc019c15b70}, 0xc019c15ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc040834f00}, {0x4e9d7a0, 0xc0af0c0c00}, 0xc01d037620, 0xc03748cd68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc040834f00}, 0xc0155dc300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08c0c99e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08c0c99e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219753676 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea3040, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06296b000, 0xc017538800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06296b000, {0xc017538800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06296b000, {0xc017538800, 0x735, 0xc05a27fe00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc041bc5e00, {0xc017538800, 0x7eda79, 0xc017538805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc031e20060, {0xc017538800, 0xc05edacc60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0731135d8, {0xc017538800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc07afe73f8, {0x631b900, 0xc0731135d8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc07afe7180, {0x7fe42ab59310, 0xc031e20060}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc07afe7180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc07afe7180, {0xc05b25b000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc08526a798, {0xc05b25b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc026184b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc026184b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0261850e0, {0x634aac0, 0xc026184b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc026184a80, 0xc08c0ffa10, 0xc05edad42b, 0xc05edad42d, {0x635bec0, 0xc00eedf7a0}, 0xc000d6ae00, 0xc00f636c88, 0xc05edad429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc026184a80, {0x63c85b8, 0xc061ff98f0}, 0xc08c0ffa00, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc051601021, 0x8}}, 0x0, 0xc061ff9830, 0xc061ff9860, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc061ff98f0}, {0x643d810, 0xc031e20060}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc099ecb560}, {0x643d810, 0xc031e20060}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc099ecb560}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808867 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06c800, {0x63c85b8, 0xc035484780}, 0xc0911c0900, 0xc0749c0640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc035484780}, 0xc0911c0900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105fd3e432, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105fd3e432, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0354846f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0354846f0}, {0xc00590f3f8, 0x660089}, 0xc00590f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0354846f0}, {0x736d726f6674616c}, 0xc0911c0700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc035484630}, 0xc0911c0700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}, 0xc082c40a20, 0xc0452fe168) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}, 0x3, 0xc0b428e380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00590f980}}}, {0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}, 0xc082c40a20, 0xc0b428e380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc035484630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc035484630}, {0x40, 0xc00590fb70}, 0xc00590fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc035484630}, {0x4e9d7a0, 0xc0911c0700}, 0xc082c40a20, 0xc0452fe168) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc035484630}, 0xc036fecd80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0193f17a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0193f17a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 207017360 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3e0400, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc062b53500, 0xc0798a1800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc062b53500, {0xc0798a1800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc062b53500, {0xc0798a1800, 0x735, 0xc058695b80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01b3098d0, {0xc0798a1800, 0x7eda79, 0xc0798a1805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03e80c660, {0xc0798a1800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc044b4aff0, {0xc0798a1800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc057e3e5f8, {0x631b900, 0xc044b4aff0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc057e3e380, {0x7fe42ab59310, 0xc03e80c660}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc057e3e380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc057e3e380, {0xc08462a000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03b1ec150, {0xc08462a000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc084628088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc084628088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc084628660, {0x634aac0, 0xc084628088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc084628000, 0xc02ef92510, 0xc089d5b42b, 0xc089d5b42d, {0x635bec0, 0xc063238fc0}, 0xc000d6ae00, 0xc040b17f78, 0xc089d5b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc084628000, {0x63c85b8, 0xc052c14360}, 0xc02ef92500, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc084626072, 0x8}}, 0x0, 0xc052c142d0, 0xc052c14300, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc052c14360}, {0x643d810, 0xc03e80c660}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0493a8930}, {0x643d810, 0xc03e80c660}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0493a8930}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803693 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00cddba40, {0x63c85b8, 0xc04f4c60f0}, 0xc037bca888, 0x1, 0xc0734ff0a8, 0x519bbc, 0xc0436a9da0, 0xc04557b260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04f4c60f0}, 0x120, 0xc0734ff230, 0x436a9d40, 0x0, 0x660069, 0xc0436a9da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00cddba40, {0x63c85b8, 0xc04f4c60f0}, 0xc0436a9d40, {{{{0x0, 0x0}, {0xc090f2bc20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f4c60f0}, 0x6352410, 0x343308f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc011c0c960, {0x63c85b8, 0xc04f4c60f0}, {0xc0436a9c20, {0x174301102edfba16, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc011c0c960, {0x63c85b8, 0xc04f4c60f0}, 0xc04d818780, 0xc0734ff9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f4c60f0}, 0x660089, {0xc0436a9c20, {0x174301102edfba16, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0122d9400, {0x63c85b8, 0xc04f4c60f0}, 0xc08af95a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102edfba16, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102edfba16, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f4c6030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f4c6030}, {0xc0735013f8, 0x660089}, 0xc073501400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f4c6030}, {0x4d939c}, 0xc08af95800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044947f80}, 0xc08af95800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}, 0xc061a12160, 0xc0aba0de78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}, 0x3, 0xc02800a480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc073501980}}}, {0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}, 0xc061a12160, 0xc02800a480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044947f80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044947f80}, {0x40, 0xc073501b70}, 0xc073501ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044947f80}, {0x4e9d7a0, 0xc08af95800}, 0xc061a12160, 0xc0aba0de78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044947f80}, 0xc01b6a31a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0436a9b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0436a9b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807825 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc078faca20}, 0xc0aef6d200, 0xc046884fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc078faca20}, 0xc0aef6d200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105988909c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105988909c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078fac7b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078fac7b0}, {0xc00f4733f8, 0x660089}, 0xc00f473400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078fac7b0}, {0x0}, 0xc0aef6cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078fac720}, 0xc0aef6cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}, 0xc06be60920, 0xc030d2db18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}, 0x3, 0xc064bad200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f473980}}}, {0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}, 0xc06be60920, 0xc064bad200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc078fac720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078fac720}, {0x40, 0xc00f473b70}, 0xc00f473ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc078fac720}, {0x4e9d7a0, 0xc0aef6cf00}, 0xc06be60920, 0xc030d2db18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc078fac720}, 0xc029deca80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc073352900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc073352900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811230 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b9a2f00, {0x63c85b8, 0xc01b20b680}, 0xc0b3d4ad08, 0x1, 0x0, 0x1, 0xc0487a9ec0, 0xc01c595a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01b20b680}, 0x120, 0xc028512728, 0x487a9e60, 0x0, 0x660069, 0xc0487a9ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b9a2f00, {0x63c85b8, 0xc01b20b680}, 0xc0487a9e60, {{{{0x0, 0x0}, {0xc0a833db00, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01b20b680}, 0x6352410, 0x6847e0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0fbb30, {0x63c85b8, 0xc01b20b680}, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0fbb30, {0x63c85b8, 0xc01b20b680}, 0xc06b47bf40, 0xc06318aee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01b20b680}, 0x660089, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9bd400, {0x63c85b8, 0xc01b20b680}, 0xc049b40b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b20b410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b20b410}, {0xc0ad83c8f0, 0x660089}, 0xc0ad83c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b20b410}, {0xc00102e4e0}, 0xc049b40a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b20b380}, 0xc049b40a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc044fe73b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219699992 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0309e0800, {0x63c85b8, 0xc061cbad50}, 0xc04e15ca00, 0xc0489f4b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0309e0800, {0x63c85b8, 0xc061cbad50}, 0xc04e15ca00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011070203ed7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011070203ed7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc061cbacc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc061cbacc0}, {0xc0949327f8, 0x660089}, 0xc094932800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc061cbacc0}, {0xc00102e4e0}, 0xc04e15c900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc061cbac60}, 0xc04e15c900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc01e113970, {0x63c85b8, 0xc061cbab10}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc060c02600, {0x63c85b8, 0xc061cbab10}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc060c02600, {0x63c8510, 0xc036d82f00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc060c02600, {0x63c8510, 0xc036d82f00}, {0x63f7db8, 0x9bf6f48}, 0xc094937088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc060c02600, {0x63c85b8, 0xc023deb950}, {{0x63f7db8, 0x9bf6f48}, {0xc084405606, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc060c02600, {0x63c85b8, 0xc023deb950}, {{0x63f7db8, 0x9bf6f48}, {0xc084405606, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc084405606, 0x6}, 0x0, 0x0}, {0x1a1b31bc, 0xedb7a8350, 0x0}, {0x1a1b8752, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc060c02600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc060c02600, {0x63c8510, 0xc0324ad200}, 0xc058bd3dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc056004360) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0324ad200}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc056004360) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811246 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e099000, {0x63c85b8, 0xc042c1d830}, 0xc0625abd00, 0xc07d31e460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc042c1d830}, 0xc0625abd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107bade4d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107bade4d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042c1d770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042c1d770}, {0xc085faa6d8, 0x660089}, 0xc085faa6e0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042c1d770}, {0xc00102e4e0}, 0xc0625abc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042c1d710}, 0xc0625abc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc07b5a8790, {0x63c85b8, 0xc042c1d590}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc040b2c600, {0x63c85b8, 0xc042c1d590}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc040b2c600, {0x63c8510, 0xc01705f100}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc040b2c600, {0x63c8510, 0xc01705f100}, 0xc01a416ad0, {0x7fe42b790cf0, 0xc04e951140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc040b2c600, {0x63c85b8, 0xc04d7f6960}, {{0x63f88f8, 0xc01a416ad0}, {0xc05b21e68a, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc040b2c600, {0x63c85b8, 0xc04d7f6960}, {{0x63f88f8, 0xc01a416ad0}, {0xc05b21e68a, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc01a416ad0}, {0xc05b21e68a, 0x23}, 0x0, 0x0}, {0x25442c89, 0xedb7a8350, 0x0}, {0x2544313a, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc040b2c600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc040b2c600, {0x63c8510, 0xc03ba08240}, 0xc07ca6fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc020495a10) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03ba08240}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc020495a10) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801768 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc094fba548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc094fba5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07d5f11f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0b1af4700, {0xc022abb180, 0x2b, 0x77}, {0xc015c134a0, 0x2b, 0x48}, 0x1, 0x11) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07d5f1088, {0xc022abb180, 0x1, 0x48}, {0xc015c134a0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d5f1088, {0xc022abb180, 0x2b, 0x77}, {0xc015c134a0, 0x2b, 0x48}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d5f07c0, {0xc015c134a0, 0x48, 0x77}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d5f07c0, {0xc022abb180, 0x2b, 0xc015c134a0}, {0xc015c134a0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d5f0500, {0xc015c134a0, 0x2b, 0x48}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc052cebb88, {{0xc08c500db0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc03b444d00, {{0xc08c500db0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0553f9400, {0x63c85b8, 0xc039901b90}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc039901b90}, {0x64a6560, 0xc03b444d00}, {0xc08c500db0, 0x41040b10a8a1ea04, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc039901b90}, {0x6466ed8, 0xc052cebb80}, {0xc08c500db0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9aaf8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0472ce100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020d8f400, {0x63c85b8, 0xc039901b90}, 0xc0472ce100, 0xc0293bf860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020d8f400, {0x63c85b8, 0xc039901b90}, 0xc0472ce100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039901b00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039901b00}, {0xc0abc4a8f0, 0x660089}, 0xc0abc4a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039901b00}, {0x24}, 0xc0472ce000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039901aa0}, 0xc0472ce000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618cad50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802676 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x69, 0xc09a25a548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc09a25a5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc086338cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc070fb8700, {0xc03cd364d0, 0x2b, 0x6d}, {0xc03cd36fc0, 0x2b, 0x6d}, 0x1, 0x8c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc086338b88, {0xc03cd364d0, 0x1, 0x6d}, {0xc03cd36fc0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc086338b88, {0xc03cd364d0, 0x2b, 0x6d}, {0xc03cd36fc0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0863382c0, {0xc03cd36fc0, 0x6d, 0x6d}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0863382c0, {0xc03cd364d0, 0x2b, 0xc03cd36fc0}, {0xc03cd36fc0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc086338000, {0xc03cd36fc0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05a2d9608, {{0xc0314f0570, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04eeb9c00, {{0xc0314f0570, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f958800, {0x63c85b8, 0xc04d00c720}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04d00c720}, {0x64a6560, 0xc04eeb9c00}, {0xc0314f0570, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04d00c720}, {0x6466ed8, 0xc05a2d9600}, {0xc0314f0570, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c77f878}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04d4c8d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4d000, {0x63c85b8, 0xc04d00c720}, 0xc04d4c8d00, 0xc06e2e4640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4d000, {0x63c85b8, 0xc04d00c720}, 0xc04d4c8d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d00c660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d00c660}, {0xc0d68368f0, 0x660089}, 0xc0d68368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d00c660}, {0xc00102e4e0}, 0xc04d4c8c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d00c5a0}, 0xc04d4c8c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220d50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802663 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0200d5000, {0x63c85b8, 0xc04e498d50}, 0xc06adfce00, 0xc052f09d60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0200d5000, {0x63c85b8, 0xc04e498d50}, 0xc06adfce00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024b4bec8, 0x0, 0x0}, 0xc0b0ae0140, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024b4bec8, 0x0, 0x0}, 0xc0b0ae0140, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e498cc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e498cc0}, {0xc076589918, 0x660089}, 0xc076589920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e498cc0}, {0x24}, 0xc06adfc400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e498c60}, 0xc06adfc400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc01bf2cae0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04e498bd0}, {0xc00dfd36b0, 0x24}, 0x46c7c8901, 0xc01526ad40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc06f0bea80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04e498bd0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812274 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc019482ff0}, 0xc017f11110, 0x1, 0x7080181b383bb3, 0x37e11d600, 0xc09fd4cde0, 0xc03efd5920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc019482ff0}, 0x120, 0xc027705230, 0x9fd4cd80, 0x0, 0x660069, 0xc09fd4cde0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc019482ff0}, 0xc09fd4cd80, {{{{0x0, 0x0}, {0xc08f7a4900, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc019482ff0}, 0x6352410, 0x3abf6348) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc019482ff0}, {0xc09fd4cc60, {0x174301107a113c4d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc019482ff0}, 0xc02160d180, 0xc0277059f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc019482ff0}, 0x660089, {0xc09fd4cc60, {0x174301107a113c4d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc019482ff0}, 0xc07d476100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a113c4d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a113c4d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc019482f60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019482f60}, {0xc0277073f8, 0x660089}, 0xc027707400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc019482f60}, {0x7fe3489cb978}, 0xc0503f3f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc019482e70}, 0xc0503f3f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}, 0xc01c0225e0, 0xc017f110c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}, 0x3, 0xc06541d540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc027707980}}}, {0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}, 0xc01c0225e0, 0xc06541d540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc019482e70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019482e70}, {0x40, 0xc027707b70}, 0xc027707ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc019482e70}, {0x4e9d7a0, 0xc0503f3f00}, 0xc01c0225e0, 0xc017f110c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc019482e70}, 0xc02b523980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc09fd4cb40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc09fd4cb40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803598 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02debacc0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219757209 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff36480, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc063f44f00, 0xc0391ea000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc063f44f00, {0xc0391ea000, 0x2b68, 0x2b68}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc063f44f00, {0xc0391ea000, 0x2b68, 0xc08b578a00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc04831cb70, {0xc0391ea000, 0x7eda79, 0xc0391ea005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc038c9fd40, {0xc0391ea000, 0xc077964c60, 0x2b68}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc075d0edf8, {0xc0391ea000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc067d16cf8, {0x631b900, 0xc075d0edf8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc067d16a80, {0x7fe42ab59310, 0xc038c9fd40}, 0x2b68) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc067d16a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc067d16a80, {0xc0997bc000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0725508e8, {0xc0997bc000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc02670b588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc02670b588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc02670bb60, {0x634aac0, 0xc02670b588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc02670b500, 0xc0435348b0, 0xc0840ef42b, 0xc0840ef42d, {0x635bec0, 0xc060d9ebd0}, 0xc000d6ae00, 0xc09b75ef18, 0xc0840ef429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc02670b500, {0x63c85b8, 0xc0429729f0}, 0xc0435348a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0b4288037, 0x8}}, 0x0, 0xc042972990, 0xc0429729c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0429729f0}, {0x643d810, 0xc038c9fd40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03ba2fce0}, {0x643d810, 0xc038c9fd40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03ba2fce0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812690 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219770122 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4d598, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc075fbb700, 0xc00c326000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc075fbb700, {0xc00c326000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc075fbb700, {0xc00c326000, 0x5, 0xc09b6c8bc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc011fab440, {0xc00c326000, 0x7eda79, 0x6}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06af14240, {0xc00c326000, 0x4bf403, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc01dc60000, {0xc00c326000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc07349fe78, {0x631b900, 0xc01dc60000}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc07349fc00, {0x7fe42ab59310, 0xc06af14240}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc07349fc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc07349fc00, {0xc0b4501000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc060c3fcc8, {0xc0b4501000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0bc3d5588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0bc3d5588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0bc3d5b60, {0x634aac0, 0xc0bc3d5588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0bc3d5500, 0xc06b537c00, 0xc09b6c942b, 0xc09b6c942d, {0x635bec0, 0xc0a20aefc0}, 0xc000d6ae00, 0xc0bcfd6f38, 0xc09b6c9429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0bc3d5500, {0x63c85b8, 0xc064807f80}, 0xc06b537bf0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc039341047, 0x8}}, 0x0, 0xc064807e30, 0xc064807f50, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc064807f80}, {0x643d810, 0xc06af14240}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0c917e1b0}, {0x643d810, 0xc06af14240}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0c917e1b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809004 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc06ee63320}, 0xc06c870690, 0x1, 0xc005a61ff0, 0x519bbc, 0xc06eb536e0, 0xc08e80d500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06ee63320}, 0x1e0, 0xc005a62178, 0x6eb53680, 0x0, 0x660069, 0xc06eb536e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc06ee63320}, 0xc06eb53680, {{{{0x0, 0x0}, {0xc07d95cfc0, 0x2c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06ee63320}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc06ee63320}, {0xc0873258c0, {0x1743011061978db5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc06ee63320}, 0xc070dc1540, 0xc0766b2938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06ee63320}, 0x660089, {0xc0873258c0, {0x1743011061978db5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc06ee63320}, 0xc077728d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061978db5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061978db5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ee63230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ee63230}, {0xc0766b4340, 0x660089}, 0xc0766b4348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ee63230}, {0x24}, 0xc044867f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06ee631d0}, 0xc044867f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc027c903f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811069 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021ad3040, {0x63c85b8, 0xc028558450}, 0xc050aa3050, 0x1, 0xc001ccd0a8, 0x519bbc, 0xc042e96940, 0xc077fcec00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0d6108480, {0x63c85b8, 0xc028558450}, 0xc042e96940, 0xc001ccd1a8, 0x30, 0x40, 0x1, 0xc042e96940, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021ad3040, {0x63c85b8, 0xc028558450}, 0xc042e96700, {{{{0xc04afc9680, 0x1}, {0xc09cc32b40, 0x1}}, {{0xc03a0bc5a0, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc028558450}, 0x63521a0, 0x202c43a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a31b80, {0x63c85b8, 0xc028558450}, {0xc00f03aea0, {0x174301100ff8dd25, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a31b80, {0x63c85b8, 0xc028558450}, 0xc0565fafa0, 0xc001ccd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc028558450}, 0x660089, {0xc00f03aea0, {0x174301100ff8dd25, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c08400, {0x63c85b8, 0xc028558450}, 0xc08b2e7d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100ff8dd25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100ff8dd25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0285583c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0285583c0}, {0xc001ccf3f8, 0x660089}, 0xc001ccf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0285583c0}, {0x3feffffe0d3f9456}, 0xc08b2e7b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc028558210}, 0xc08b2e7b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}, 0xc0148dea20, 0xc03a288ff0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}, 0x3, 0xc01f41d500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc001ccf980}}}, {0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}, 0xc0148dea20, 0xc01f41d500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc028558210}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028558210}, {0x40, 0xc001ccfb70}, 0xc001ccfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc028558210}, {0x4e9d7a0, 0xc08b2e7b00}, 0xc0148dea20, 0xc03a288ff0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc028558210}, 0xc02f03c960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00f03ad80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00f03ad80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806964 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc0938b1170}, 0xc02e15df80, 0x1, 0x708017f12c7f2d, 0x37e11d600, 0xc0d3ca64e0, 0xc066b83ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0938b1170}, 0x120, 0xc004ed5230, 0xd3ca6480, 0x0, 0x660069, 0xc0d3ca64e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc0938b1170}, 0xc0d3ca6480, {{{{0x0, 0x0}, {0xc03cace000, 0x1c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0938b1170}, 0x6352410, 0x25ab65c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc0938b1170}, {0xc0d3ca6240, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc0938b1170}, 0xc0761d0d20, 0xc004ed59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0938b1170}, 0x660089, {0xc0d3ca6240, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc0938b1170}, 0xc078ff9100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0938b10e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0938b10e0}, {0xc004ed73f8, 0x660089}, 0xc004ed7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0938b10e0}, {0x0}, 0xc078ff8f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0938b1050}, 0xc078ff8f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}, 0xc08e4ebe00, 0xc02e15df50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}, 0x3, 0xc08f432d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004ed7980}}}, {0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}, 0xc08e4ebe00, 0xc08f432d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0938b1050}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0938b1050}, {0x40, 0xc004ed7b70}, 0xc004ed7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0938b1050}, {0x4e9d7a0, 0xc078ff8f00}, 0xc08e4ebe00, 0xc02e15df50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0938b1050}, 0xc067e10720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0d3ca6120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0d3ca6120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219756076 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc02ed1ee70, {0x63c85b8, 0xc042ef9fb0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0511a7900, {0x63c85b8, 0xc042ef9fb0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0511a7900, {0x63c8510, 0xc05bdeb340}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc0511a7900, {0x63c8510, 0xc05bdeb340}, 0xc07b6f7a50, {0x7fe42b790cf0, 0xc0a3779bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0511a7900, {0x63c85b8, 0xc0133e6c60}, {{0x63f88f8, 0xc07b6f7a50}, {0xc000036c89, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0511a7900, {0x63c85b8, 0xc0133e6c60}, {{0x63f88f8, 0xc07b6f7a50}, {0xc000036c89, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc07b6f7a50}, {0xc000036c89, 0x23}, 0x0, 0x0}, {0x7ab26f0, 0xedb7a8350, 0x0}, {0x7ab2cd3, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0511a7900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0511a7900, {0x63c8510, 0xc0603d7740}, 0xc07c441dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04f0daa50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0603d7740}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04f0daa50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219143258 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc000f2cb40, {0x63c85b8, 0xc04e8e7a40}, 0xc063c17ea0, {0xc01d5263d0, 0x10}, 0xc02a56e180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1462 +0x2c5 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1({0x63c85b8, 0xc04e8e7a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1380 +0x8d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802591 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04181eb10}, 0xc05b6e1770, 0x1, 0xc05462f0a8, 0x519bbc, 0xc0cbbcd5c0, 0xc047c81140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04181eb10}, 0x120, 0xc05462f230, 0xcbbcd560, 0x0, 0x660069, 0xc0cbbcd5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04181eb10}, 0xc0cbbcd560, {{{{0x0, 0x0}, {0xc083361440, 0xcd}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04181eb10}, 0x6352410, 0x6492b610) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04181eb10}, {0xc0cbbcd440, {0x1743011024488cdb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04181eb10}, 0xc07076b360, 0xc05462f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04181eb10}, 0x660089, {0xc0cbbcd440, {0x1743011024488cdb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04181eb10}, 0xc04c13e500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024488cdb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024488cdb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04181ea80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04181ea80}, {0xc0546313f8, 0x660089}, 0xc054631400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04181ea80}, {0xc0015221e0}, 0xc04c13e200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04181e960}, 0xc04c13e200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}, 0xc0533a3a60, 0xc0571d5590) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}, 0x3, 0xc042d2cc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc054631980}}}, {0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}, 0xc0533a3a60, 0xc042d2cc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04181e960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04181e960}, {0x40, 0xc054631b70}, 0xc054631ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04181e960}, {0x4e9d7a0, 0xc04c13e200}, 0xc0533a3a60, 0xc0571d5590) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04181e960}, 0xc01e3f5f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0cbbcd320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0cbbcd320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811185 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ffbe500, {0x63c85b8, 0xc05ca38bd0}, 0xc035a04948, 0x1, 0xc0199e50a8, 0x519bbc, 0xc0ad7c6960, 0xc07770f6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05ca38bd0}, 0x120, 0xc0199e5230, 0xad7c6900, 0x0, 0x660069, 0xc0ad7c6960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ffbe500, {0x63c85b8, 0xc05ca38bd0}, 0xc0ad7c6900, {{{{0x0, 0x0}, {0xc05bd48900, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05ca38bd0}, 0x6352410, 0x3824c1b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f552c80, {0x63c85b8, 0xc05ca38bd0}, {0xc0ad7c67e0, {0x174301106f0dab1c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f552c80, {0x63c85b8, 0xc05ca38bd0}, 0xc02160d040, 0xc0199e59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05ca38bd0}, 0x660089, {0xc0ad7c67e0, {0x174301106f0dab1c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ffb7c00, {0x63c85b8, 0xc05ca38bd0}, 0xc0664b7b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f0dab1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f0dab1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05ca38b40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05ca38b40}, {0xc0199e73f8, 0x660089}, 0xc0199e7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05ca38b40}, {0x0}, 0xc0664b7900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05ca38ab0}, 0xc0664b7900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}, 0xc040f5d660, 0xc0bba627b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}, 0x3, 0xc011871780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0199e7980}}}, {0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}, 0xc040f5d660, 0xc011871780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05ca38ab0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05ca38ab0}, {0x40, 0xc0199e7b70}, 0xc0199e7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05ca38ab0}, {0x4e9d7a0, 0xc0664b7900}, 0xc040f5d660, 0xc0bba627b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05ca38ab0}, 0xc022a0bb60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0ad7c66c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0ad7c66c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219000390 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc062e59e90, 0x24) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0e5b24000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0e5b24660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0d5864600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0d5864600, {0x63c8510, 0xc062e59ec0}, 0xc0c553bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0411ee5d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc062e59ec0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0411ee5d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810168 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0214da000, {0x63c85b8, 0xc0531c4900}, 0xc0292c5f00, 0xc07783b680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc0531c4900}, 0xc0292c5f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106be9b254, 0x0, 0x0}, 0xc0530075b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106be9b254, 0x0, 0x0}, 0xc0530075b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0531c4870}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0531c4870}, {0xc01aff13f8, 0x660089}, 0xc01aff1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0531c4870}, {0x0}, 0xc0292c5d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0531c4780}, 0xc0292c5d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}, 0xc02906a360, 0xc04ebf6df8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}, 0x3, 0xc0132db180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aff1980}}}, {0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}, 0xc02906a360, 0xc0132db180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0531c4780}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0531c4780}, {0x40, 0xc01aff1b70}, 0xc01aff1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0531c4780}, {0x4e9d7a0, 0xc0292c5d00}, 0xc02906a360, 0xc04ebf6df8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0531c4780}, 0xc046e4bd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06b2478c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06b2478c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810149 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc056358d20}, 0xc02a8dd518, 0x1, 0xc00eb450a8, 0x519bbc, 0xc0525ed480, 0xc04cdc5bc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04d87c900, {0x63c85b8, 0xc056358d20}, 0xc04e46ab40, 0xc00eb451a8, 0x525ed480, 0x0, 0x0, 0xc0525ed480, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc056358d20}, 0xc04ebfa420, {{{{0xc079db7b00, 0x46}, {0xc00449d8c0, 0x76}}, {{0xc06f5fcd80, 0x98}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc056358d20}, 0x63521a0, 0x43391dd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc056358d20}, {0xc07bdf98c0, {0x1743010ff0cc31e6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc056358d20}, 0xc02e47c5a0, 0xc00eb459f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc056358d20}, 0x660089, {0xc07bdf98c0, {0x1743010ff0cc31e6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc056358d20}, 0xc0551b3300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff0cc31e6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff0cc31e6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056358c90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056358c90}, {0xc00eb473f8, 0x660089}, 0xc00eb47400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056358c90}, {0x7fe3488b59c0}, 0xc0551b2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056358bd0}, 0xc0551b2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}, 0xc01696c700, 0xc06fab1860) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}, 0x3, 0xc00ff06f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00eb47980}}}, {0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}, 0xc01696c700, 0xc00ff06f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056358bd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056358bd0}, {0x40, 0xc00eb47b70}, 0xc00eb47ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056358bd0}, {0x4e9d7a0, 0xc0551b2e00}, 0xc01696c700, 0xc06fab1860) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056358bd0}, 0xc04ebfa2a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07bdf9680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07bdf9680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795598 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe79c78, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0b8632880, 0xc00a8e7000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0b8632880, {0xc00a8e7000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0b8632880, {0xc00a8e7000, 0x735, 0xc04462a780}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0381d2c78, {0xc00a8e7000, 0x7eda79, 0xc00a8e7005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0d3a35e00, {0xc00a8e7000, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc041bf2420, {0xc00a8e7000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0836dccf8, {0x631b900, 0xc041bf2420}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0836dca80, {0x7fe42ab59310, 0xc0d3a35e00}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0836dca80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0836dca80, {0xc064ab6000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc022258510, {0xc064ab6000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc036df7588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc036df7588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc036df7b60, {0x634aac0, 0xc036df7588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc036df7500, 0xc01e8c6350, 0xc03a60942b, 0xc03a60942d, {0x635bec0, 0xc07c1742d0}, 0xc000d6ae00, 0xc054cfb1c8, 0xc03a609429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc036df7500, {0x63c85b8, 0xc040b05e30}, 0xc01e8c6300, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc03be7e03e, 0x8}}, 0x0, 0xc040b05dd0, 0xc040b05e00, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc040b05e30}, {0x643d810, 0xc0d3a35e00}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07df54ba0}, {0x643d810, 0xc0d3a35e00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07df54ba0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812679 [runnable]: sync.runtime_SemacquireMutex(0xc0409f8fa8, 0x6c, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x4, 0xc00a31e900) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01e9c1080, {0x804f921, 0x466a}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc00a31e900, {0xffffffffffffff15, 0xc0409f91d8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc00a31e900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc00a31e900, {0xc0bbb96000, 0x48, 0x74}, {0xc03a0ae4d0, 0x48, 0x6e}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092fee398, {0xc0bbb96000, 0x48, 0x74}, {0xc03a0ae4d0, 0x48, 0x6e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092feccc0, {0xc03a0ae4d0, 0x0, 0xc0409f94e0}, 0x121ea30, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092feccc0, {0xc0bbb96000, 0x48, 0x6e}, {0xc03a0ae4d0, 0x47, 0x50}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092feca00, {0xc03a0ae4d0, 0x48, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc00e00cc08, {{0xc02fa4d270, 0x47, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc045891100, {{0xc02fa4d270, 0x47, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05ca12000, {0x63c85b8, 0xc0752085d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0752085d0}, {0x64a6560, 0xc045891100}, {0xc02fa4d270, 0x0, 0x50}, {0x174301109ec02cab, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0752085d0}, {0x6466ed8, 0xc00e00cc00}, {0xc02fa4d270, 0x47, 0x50}, {0x174301109ec02cab, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0240afdd0}, {{0x174301109ec02cab, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109ec02cab, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0711f0d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bc8d91ab, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc000e4e800, {0x63c85b8, 0xc0752085d0}, 0xc0711f0d00, 0xc050934e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000e4e800, {0x63c85b8, 0xc0752085d0}, 0xc0711f0d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109ec02cab, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109ec02cab, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075208540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075208540}, {0xc0409ff3f8, 0x660089}, 0xc0409ff400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075208540}, {0x1}, 0xc0711f0b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0752084b0}, 0xc0711f0b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}, 0xc00abd3a40, 0xc017daf380) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}, 0x3, 0xc06b3d7300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0409ff980}}}, {0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}, 0xc00abd3a40, 0xc06b3d7300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0752084b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0752084b0}, {0x40, 0xc0409ffb70}, 0xc0409ffab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0752084b0}, {0x4e9d7a0, 0xc0711f0b00}, 0xc00abd3a40, 0xc017daf380) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0752084b0}, 0xc02e5a1320, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00de6b7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc00de6b7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219793019 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0654c4f60}, 0xc02457acb0, {0x646cc48, 0xc0654c4fc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0654c4f60}, 0xedb7a834e, 0xc02ceeefc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0654c4f60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0206b33b0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc02457aef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0206b33b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804948 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).ResolveIndeterminateCommit(0xc00d502dc0, {0x63c85b8, 0xc022790690}, 0xc0518be7e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:100 +0x2f7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleIndeterminateCommitError(0xc01e9c9630, {0x63c85b8, 0xc022790690}, 0xc07c08be00, 0xc08b561740, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:743 +0x4f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f506c00, {0x63c85b8, 0xc022790690}, 0xc07c08be00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:503 +0x8d3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110147abbaf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110147abbaf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022790600}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022790600}, {0xc07526f3f8, 0x660089}, 0xc07526f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022790600}, {0x7fe3f723dbc8}, 0xc07c08bc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022790390}, 0xc07c08bc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}, 0xc06649a660, 0xc031050390) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}, 0x3, 0xc05969a3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07526f980}}}, {0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}, 0xc06649a660, 0xc05969a3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022790390}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022790390}, {0x40, 0xc07526fb70}, 0xc07526fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}, 0x203016, 0x203016) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022790390}, {0x4e9d7a0, 0xc07c08bc00}, 0xc06649a660, 0xc031050390) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022790390}, 0xc018c9dd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f39aea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f39aea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812799 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801672 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc020f00e10}, 0xc0479f0768, 0x1, 0xc054e8b0a8, 0x519bbc, 0xc0645d16e0, 0xc057e16f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc020f00e10}, 0x120, 0xc054e8b230, 0x645d1680, 0x0, 0x660069, 0xc0645d16e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc020f00e10}, 0xc0645d1680, {{{{0x0, 0x0}, {0xc09ed7f8c0, 0xea}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc020f00e10}, 0x6352410, 0x76af6790) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc020f00e10}, {0xc0645d1560, {0x1743011019f1e7a2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc020f00e10}, 0xc048adff40, 0xc054e8b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc020f00e10}, 0x660089, {0xc0645d1560, {0x1743011019f1e7a2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc020f00e10}, 0xc052a52500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019f1e7a2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019f1e7a2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020f00d80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020f00d80}, {0xc054e8d3f8, 0x660089}, 0xc054e8d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020f00d80}, {0x0}, 0xc052a52300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020f00cc0}, 0xc052a52300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}, 0xc07ab7f600, 0xc09d75c438) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}, 0x3, 0xc0cb24da80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc054e8d980}}}, {0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}, 0xc07ab7f600, 0xc0cb24da80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020f00cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020f00cc0}, {0x40, 0xc054e8db70}, 0xc054e8dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020f00cc0}, {0x4e9d7a0, 0xc052a52300}, 0xc07ab7f600, 0xc09d75c438) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020f00cc0}, 0xc042d263c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc054ea5200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc054ea5200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811143 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035040, {0x63c85b8, 0xc018600120}, 0xc03dfa9800, 0x1, 0xc084b95920, 0x519bbc, 0xc019bbbc80, 0xc0d19a9020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc03e9cdc20, {0x63c85b8, 0xc018600120}, 0xc03e9cdc78, 0xc084b95a20, 0x41, 0x60, 0xffffffffffffffff, 0xc019bbbc80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035040, {0x63c85b8, 0xc018600120}, 0xc019bbbc20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc077e2f680, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc018600120}, 0x63523b0, 0x3824ca10) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd3b0, {0x63c85b8, 0xc018600120}, {0x0, {0x174301107905f84a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd3b0, {0x63c85b8, 0xc018600120}, 0xc07cc50280, 0xc084b96268) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc018600120}, 0x660089, {0x0, {0x174301107905f84a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc018600120}, 0xc06cbd3100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ec8df20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ec8df20}, {0xc01f7c7c70, 0x660089}, 0xc01f7c7c78) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ec8df20}, {0xc046386ae0}, 0xc06cbd2f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ec8dec0}, 0xc06cbd2f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x174301107905f84a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc046386ea0}, 0x0, 0xc0d10b0c00) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc046386ea0}, 0x19) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0078dea50, {0x63c85b8, 0xc046386ea0}, 0xc01f7ca6b0, {{0x174301107905e2dd, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:392 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0x0, {0x63c85b8, 0xc046386ea0}, 0xc04474e398, {{0x174301107905e2dd, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x165 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc02dfdd400, {0x63c85b8, 0xc046386ea0}, {0xc01cd6c7e0, {0x1743011072fbcd8c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:499 +0x533 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc046386ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc02dfdd400, {0x63c85b8, 0xc046386ea0}, {0xc01cd6c7e0, {0x1743011072fbcd8c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd3b0, {0x63c85b8, 0xc046386ea0}, 0xc057f20d20, 0xc01f7cb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046386ea0}, 0x660089, {0xc01cd6c7e0, {0x1743011072fbcd8c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc046386ea0}, 0xc072b39500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011072fbcd8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011072fbcd8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046386de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046386de0}, {0xc01f7cd3f8, 0x660089}, 0xc01f7cd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046386de0}, {0x7fe32629fc10}, 0xc072b39300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046386c90}, 0xc072b39300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}, 0xc01ed826e0, 0xc03a388288) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}, 0x3, 0xc047938d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f7cd980}}}, {0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}, 0xc01ed826e0, 0xc047938d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046386c90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046386c90}, {0x40, 0xc01f7cdb70}, 0xc01f7cdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046386c90}, {0x4e9d7a0, 0xc072b39300}, 0xc01ed826e0, 0xc03a388288) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046386c90}, 0xc01cd8ccc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01cd6c480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01cd6c480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812744 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810087 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc048e07320}, 0xc028b35cb0, {0x646cc48, 0xc048e07380}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc048e07320}, 0xedb7a8350, 0xc0638f4120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc048e07320}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06c14e840}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc028b35ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06c14e840}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219484324 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc08638ebb0, {0x63c85b8, 0xc088d4d200}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc021892000, {0x63c85b8, 0xc088d4d200}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc021892000, {0x63c8510, 0xc07d79fa80}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc021892000, {0x63c8510, 0xc07d79fa80}, {0x63f7db8, 0x9bf6f48}, 0xc076369088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc021892000, {0x63c85b8, 0xc086e6e660}, {{0x63f7db8, 0x9bf6f48}, {0xc0869e052f, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc021892000, {0x63c85b8, 0xc086e6e660}, {{0x63f7db8, 0x9bf6f48}, {0xc0869e052f, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0869e052f, 0x6}, 0x0, 0x0}, {0x33bafa6a, 0xedb7a834f, 0x0}, {0x33baff52, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc021892000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc021892000, {0x63c8510, 0xc0558cfe80}, 0xc08b8e5dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0115fc6e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0558cfe80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0115fc6e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 214101596 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc06e9c42d0, 0x8ee) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc042eb2a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc042eb30e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05f7a7300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05f7a7300, {0x63c8510, 0xc06e9c4300}, 0xc058c69dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05a57c1b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06e9c4300}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05a57c1b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802047 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc02e07d800, {0x63c85b8, 0xc046e83980}, 0xc03f1ccf00, 0xc0a1c86440, 0xc04a06b6d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:563 +0x6af github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc046e83980}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc046e836e0}, {{0xc035a27a80, 0x24, 0x24}}, 0xc09a81dbf8) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc02e07d800, {0x63c85b8, 0xc046e836e0}, 0xc03f1ccf00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc02dfddb30, {0x63c85b8, 0xc046e836e0}, {0x0, {0x17430110268a8931, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc046e836e0}, 0xc086299cc0, 0xc07cbfe150) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046e836e0}, 0xc086299cc0, {0x0, {0x17430110268a8931, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc046e836e0}, 0xc0588e3200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046e83650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046e83650}, {0xc058e63c70, 0x660089}, 0xc058e63c78) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046e83650}, {0xc07cbfe030}, 0xc0588e3100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046e835f0}, 0xc0588e3100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x17430110268a8931, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc07cbfe270}, 0x0, 0xc0d50ab080) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc07cbfe270}, 0x19) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0078dea50, {0x63c85b8, 0xc07cbfe270}, 0xc058e666b0, {{0x17430110268a72c6, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:392 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0x0, {0x63c85b8, 0xc07cbfe270}, 0xc013734218, {{0x17430110268a72c6, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x165 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc02dfddb80, {0x63c85b8, 0xc07cbfe270}, {0xc041e27200, {0x17430110236de7cc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:499 +0x533 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc07cbfe270}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc02dfddb80, {0x63c85b8, 0xc07cbfe270}, {0xc041e27200, {0x17430110236de7cc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc07cbfe270}, 0xc033dbb0e0, 0xc058e67558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07cbfe270}, 0xc033dbb0e0, {0xc041e27200, {0x17430110236de7cc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc07cbfe270}, 0xc02754cd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110236de7cc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110236de7cc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07cbfe1e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07cbfe1e0}, {0xc058e693f8, 0x660089}, 0xc058e69400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07cbfe1e0}, {0xc0015221e0}, 0xc02754c500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07cbfe150}, 0xc02754c500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}, 0xc03c73b380, 0xc0225c8888) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}, 0x3, 0xc0407dfa00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc058e69980}}}, {0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}, 0xc03c73b380, 0xc0407dfa00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07cbfe150}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07cbfe150}, {0x40, 0xc058e69b70}, 0xc058e69ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07cbfe150}, {0x4e9d7a0, 0xc02754c500}, 0xc03c73b380, 0xc0225c8888) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07cbfe150}, 0xc037fadaa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041e26fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041e26fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219797115 [runnable]: sync.runtime_SemacquireMutex(0xc08e9a6690, 0x8f, 0x7fda4c915ec0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc08e9a6538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc08e9a6560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc05b12a900, {0xffffffffffffff15, 0xc08e9a66c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc05b12a900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc05b12a900, {0xc03fcf5a40, 0x32, 0x6d}, {0xc02e144c00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc00b264398, {0xc03fcf5a40, 0x32, 0x6d}, {0xc02e144c00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00b262cc0, {0xc02e144c00, 0x74, 0x6d}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00b262cc0, {0xc03fcf5a40, 0x74, 0xc02e144c00}, {0xc02e144c00, 0xc0783c1e10, 0x31}, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00b262a00, {0xc02e144c00, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0848ea828, {{0xc0783c1e10, 0x31, 0xd0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06bc1e200, {{0xc0369249c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc03623bc00, {0x63c85b8, 0xc01e71e1e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01e71e1e0}, {0x64a6560, 0xc06bc1e200}, {0xc0369249c0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01e71e1e0}, {0x6466ed8, 0xc0848ea580}, {0xc0369249c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc041282258}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0497d9c00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020480c00, {0x63c85b8, 0xc01e71e1e0}, 0xc0497d9c00, 0xc01cc9b180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020480c00, {0x63c85b8, 0xc01e71e1e0}, 0xc0497d9c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e71e0f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e71e0f0}, {0xc0032f88f0, 0x660089}, 0xc0032f88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e71e0f0}, {0xc00102e4e0}, 0xc0497d9a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e71e090}, 0xc0497d9a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc041f5aea0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809040 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0213bbe00, {0x63c85b8, 0xc07b6c1bf0}, 0xc061b3ac18, 0x1, 0xc02476d0a8, 0x519bbc, 0xc04c55b020, 0xc081f6e840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07b6c1bf0}, 0x120, 0xc02476d230, 0x4c55afc0, 0x0, 0x660069, 0xc04c55b020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0213bbe00, {0x63c85b8, 0xc07b6c1bf0}, 0xc04c55afc0, {{{{0x0, 0x0}, {0xc0429158c0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07b6c1bf0}, 0x6352410, 0x3102e970) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135ba90, {0x63c85b8, 0xc07b6c1bf0}, {0xc04c55aea0, {0x174301105b86b204, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135ba90, {0x63c85b8, 0xc07b6c1bf0}, 0xc02e903400, 0xc02476d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07b6c1bf0}, 0x660089, {0xc04c55aea0, {0x174301105b86b204, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214a9400, {0x63c85b8, 0xc07b6c1bf0}, 0xc07cd7b100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b86b204, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b86b204, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07b6c1b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b6c1b60}, {0xc02476f3f8, 0x660089}, 0xc02476f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07b6c1b60}, {0x322d626631382d63}, 0xc07cd7af00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07b6c1ad0}, 0xc07cd7af00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}, 0xc050616ae0, 0xc0246f9800) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}, 0x3, 0xc00e4e2ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02476f980}}}, {0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}, 0xc050616ae0, 0xc00e4e2ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07b6c1ad0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b6c1ad0}, {0x40, 0xc02476fb70}, 0xc02476fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07b6c1ad0}, {0x4e9d7a0, 0xc07cd7af00}, 0xc050616ae0, 0xc0246f9800) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07b6c1ad0}, 0xc04f016900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc034ef5b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc034ef5b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 207646453 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc06c1e5310, 0xde) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc087b30a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc087b310e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03c628000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03c628000, {0x63c8510, 0xc06c1e5340}, 0xc060e49dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02dcbd110) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06c1e5340}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02dcbd110) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811124 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc035328780, {0x63c85b8, 0xc03aaa5410}, 0xc0715c80d8, 0x1, 0xc010d6bff0, 0x519bbc, 0xc039edbc80, 0xc018a62cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03aaa5410}, 0x120, 0xc010d6c178, 0x39edbc20, 0x0, 0x660069, 0xc039edbc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc035328780, {0x63c85b8, 0xc03aaa5410}, 0xc039edbc20, {{{{0x0, 0x0}, {0xc0d6108240, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03aaa5410}, 0x6352410, 0x537e69b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05bb9e870, {0x63c85b8, 0xc03aaa5410}, {0xc039edbb00, {0x174301107264d606, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05bb9e870, {0x63c85b8, 0xc03aaa5410}, 0xc04c80b9a0, 0xc088136938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03aaa5410}, 0x660089, {0xc039edbb00, {0x174301107264d606, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0716f9400, {0x63c85b8, 0xc03aaa5410}, 0xc0716ea800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107264d606, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107264d606, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03aaa5380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03aaa5380}, {0xc088138340, 0x660089}, 0xc088138348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03aaa5380}, {0xc00102e4e0}, 0xc0716ea700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03aaa5320}, 0xc0716ea700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc07221a720}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219774532 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc012099050, 0x2) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc03d371500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc03d371b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc094af3300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc094af3300, {0x63c8510, 0xc012099080}, 0xc0bbce9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02d882440) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc012099080}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02d882440) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802117 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01decd800, {0x63c85b8, 0xc073ec09f0}, 0xc04ecefd00, 0xc077d0b720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01decd800, {0x63c85b8, 0xc073ec09f0}, 0xc04ecefd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020875c07, 0x0, 0x0}, 0xc04e8f7d50, {0x1f, 0x1f, 0x1d, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020875c07, 0x0, 0x0}, 0xc04e8f7d50, {0x1f, 0x1f, 0x1d, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1d, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073ec0960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073ec0960}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073ec0960}, {0xc00102e4e0}, 0xc04ecefc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073ec0840}, 0xc04ecefc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1d, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc0783db740}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc073ec07b0}, {0xc00dfd36b0, 0x24}, 0x46c7bf216, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc070700bd0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc073ec07b0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806910 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc0931dca50}, 0xc038af9620, 0x1, 0xc002de90a8, 0x519bbc, 0xc037fbfc80, 0xc046bdeba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0931dca50}, 0x120, 0xc002de9230, 0x37fbfc20, 0x0, 0x660069, 0xc037fbfc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc0931dca50}, 0xc037fbfc20, {{{{0x0, 0x0}, {0xc09c674d80, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0931dca50}, 0x6352410, 0x4f454cd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc0931dca50}, {0xc037fbfb00, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc0931dca50}, 0xc07bbedcc0, 0xc002de99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0931dca50}, 0x660089, {0xc037fbfb00, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc0931dca50}, 0xc0791e6700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0931dc9c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0931dc9c0}, {0xc002deb3f8, 0x660089}, 0xc002deb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0931dc9c0}, {0x0}, 0xc0791e6500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0931dc930}, 0xc0791e6500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}, 0xc08fc48c20, 0xc038af9d28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}, 0x3, 0xc0927a8480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002deb980}}}, {0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}, 0xc08fc48c20, 0xc0927a8480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0931dc930}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0931dc930}, {0x40, 0xc002debb70}, 0xc002debab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0931dc930}, {0x4e9d7a0, 0xc0791e6500}, 0xc08fc48c20, 0xc038af9d28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0931dc930}, 0xc065d022a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037fbf9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037fbf9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803822 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01edcb400, {0x63c85b8, 0xc01f6ef6e0}, 0xc05f1f6800, 0xc046ded860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc01f6ef6e0}, 0xc05f1f6800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110312c7b4c, 0x0, 0x0}, 0xc0b4a48490, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110312c7b4c, 0x0, 0x0}, 0xc0b4a48490, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01f6ef650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01f6ef650}, {0xc07526f3f8, 0x660089}, 0xc07526f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01f6ef650}, {0x7fe3e212bb70}, 0xc05f1f6600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01f6ef5c0}, 0xc05f1f6600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}, 0xc01ba258a0, 0xc0428e84b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}, 0x3, 0xc0658ac940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07526f980}}}, {0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}, 0xc01ba258a0, 0xc0658ac940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01f6ef5c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01f6ef5c0}, {0x40, 0xc07526fb70}, 0xc07526fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01f6ef5c0}, {0x4e9d7a0, 0xc05f1f6600}, 0xc01ba258a0, 0xc0428e84b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01f6ef5c0}, 0xc08659b020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04fd10000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04fd10000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803502 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc043d66570}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812778 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806912 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc0931dd3b0}, 0xc0449d2030, 0x1, 0x708017f0e0b6b3, 0x37e11d600, 0xc048d5e3c0, 0xc087a135c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0931dd3b0}, 0x120, 0xc004a13230, 0x48d5e360, 0x0, 0x660069, 0xc048d5e3c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc0931dd3b0}, 0xc048d5e360, {{{{0x0, 0x0}, {0xc03cace000, 0x1c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0931dd3b0}, 0x6352410, 0x4f454ce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc0931dd3b0}, {0xc048d5e240, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc0931dd3b0}, 0xc0678be000, 0xc004a139f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0931dd3b0}, 0x660089, {0xc048d5e240, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc0931dd3b0}, 0xc0791e6e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0931dd320}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0931dd320}, {0xc004a153f8, 0x660089}, 0xc004a15400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0931dd320}, {0x40d1c2bfb9a53870}, 0xc0791e6b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0931dd290}, 0xc0791e6b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}, 0xc08fc48d60, 0xc0449d2000) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}, 0x3, 0xc0927a8780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004a15980}}}, {0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}, 0xc08fc48d60, 0xc0927a8780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0931dd290}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0931dd290}, {0x40, 0xc004a15b06}, 0xc004a15ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0931dd290}, {0x4e9d7a0, 0xc0791e6b00}, 0xc08fc48d60, 0xc0449d2000) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0931dd290}, 0xc065d02ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc048d5e120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc048d5e120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800376 [semacquire]: sync.runtime_SemacquireMutex(0x3754d46, 0x80, 0x7fe08f7d7a20) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc05bc944e0, 0x300000002, 0xc05bc944e0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc000083000, 0xc090824420, 0x4571d1, 0x424a070) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7236e27, 0x3b81}, 0x0, 0xc04eb6c278) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04eb6c000, {0xffffffffffffff15, 0xc0908245b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc04eb6c000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc04eb6c000, {0xc024911580, 0x32, 0x77}, 0x13, 0x15) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04eb6c000, {0xc0796d1050, 0x32, 0x77}, {0xc024911580, 0x51c9860, 0xc024911580}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04eb6c000, {0xc0796d1050, 0x32, 0x88}, {0xc024911580, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0beccbe98, {0xc0796d1050, 0x32, 0x88}, {0xc024911580, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0becca7c0, {0xc024911580, 0x77, 0x88}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0becca7c0, {0xc0796d1050, 0x77, 0xc024911580}, {0xc024911580, 0xc0482a9320, 0x31}, 0x84) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0becca500, {0xc024911580, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06c56a828, {{0xc0482a9320, 0x31, 0x84}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0b58dcf00, {{0xc03ab44a80, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0590b0800, {0x63c85b8, 0xc04f807fb0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04f807fb0}, {0x64a6560, 0xc0b58dcf00}, {0xc03ab44a80, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04f807fb0}, {0x6466ed8, 0xc06c56a580}, {0xc03ab44a80, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0282d0a20}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc071b6e700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ab9800, {0x63c85b8, 0xc04f807fb0}, 0xc071b6e700, 0xc033dbbf40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ab9800, {0x63c85b8, 0xc04f807fb0}, 0xc071b6e700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f807ef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f807ef0}, {0xc0ad8388f0, 0x660089}, 0xc0ad8388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f807ef0}, {0xc00102e4e0}, 0xc071b6e600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f807e90}, 0xc071b6e600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b5f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809620 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc07a065680}, 0xc031f36798, 0x1, 0xc027b110a8, 0x519bbc, 0xc088226cc0, 0xc0741e95c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07a065680}, 0x120, 0xc027b11230, 0x88226c60, 0x0, 0x660069, 0xc088226cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc07a065680}, 0xc088226c60, {{{{0x0, 0x0}, {0xc073d510e0, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07a065680}, 0x6352410, 0x43391660) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc07a065680}, {0xc088226b40, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc07a065680}, 0xc0650994a0, 0xc027b119f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a065680}, 0x660089, {0xc088226b40, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc07a065680}, 0xc04eb51b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a065590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a065590}, {0xc027b133f8, 0x660089}, 0xc027b13400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a065590}, {0xc097081400}, 0xc04eb51900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a065500}, 0xc04eb51900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}, 0xc028ad5660, 0xc035a42858) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}, 0x3, 0xc0633e29c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc027b13980}}}, {0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}, 0xc028ad5660, 0xc0633e29c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a065500}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a065500}, {0x40, 0xc027b13b70}, 0xc027b13ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a065500}, {0x4e9d7a0, 0xc04eb51900}, 0xc028ad5660, 0xc035a42858) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a065500}, 0xc047da8ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc088226a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc088226a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811549 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc04bb9ae10}, 0xc0b8610ba0, 0x1, 0xc01096b0a8, 0x519bbc, 0xc017fbb980, 0xc05c9e1ce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0778ff440, {0x63c85b8, 0xc04bb9ae10}, 0xc0778ff4d8, 0xc01096b1a8, 0x40, 0x60, 0xffffffffffffffff, 0xc017fbb980, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc04bb9ae10}, 0xc017fbb8c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc01f212000, 0x21}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04bb9ae10}, 0x6352260, 0x54d3a1d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc04bb9ae10}, {0xc017fbb440, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc04bb9ae10}, 0xc0780499a0, 0xc01096b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04bb9ae10}, 0x660089, {0xc017fbb440, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc04bb9ae10}, 0xc08a8d2b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bb9ad20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bb9ad20}, {0xc01096d3f8, 0x660089}, 0xc01096d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bb9ad20}, {0x0}, 0xc08a8d2900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bb9ac60}, 0xc08a8d2900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}, 0xc06d764fc0, 0xc044b4a768) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}, 0x3, 0xc06e1064c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01096d980}}}, {0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}, 0xc06d764fc0, 0xc06e1064c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bb9ac60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bb9ac60}, {0x40, 0xc01096db70}, 0xc01096dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bb9ac60}, {0x4e9d7a0, 0xc08a8d2900}, 0xc06d764fc0, 0xc044b4a768) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bb9ac60}, 0xc04c4d8120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc017fbafc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc017fbafc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219150673 [runnable]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc041da8550, 0x1) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc073504660) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:332 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:329 +0x198f goroutine 219810440 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0c0a0bb80, {0x63c85b8, 0xc0780283f0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc08fa8a000, {0x63c85b8, 0xc0780283f0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc08fa8a000, {0x63c8510, 0xc011d84fc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc08fa8a000, {0x63c8510, 0xc011d84fc0}, 0xc0872b2da0, {0x7fe42b790cf0, 0xc0a05126c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc08fa8a000, {0x63c85b8, 0xc01fb68660}, {{0x63f88f8, 0xc0872b2da0}, {0xc0a5f97836, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc08fa8a000, {0x63c85b8, 0xc01fb68660}, {{0x63f88f8, 0xc0872b2da0}, {0xc0a5f97836, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0872b2da0}, {0xc0a5f97836, 0x23}, 0x0, 0x0}, {0x2808dd4b, 0xedb7a8350, 0x0}, {0x2808e35f, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08fa8a000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08fa8a000, {0x63c8510, 0xc006697cc0}, 0xc07cdf7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0470d0440) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc006697cc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0470d0440) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219150690 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc06cd01860) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 219788947 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc053047080}, 0xc04a60ecb0, {0x646cc48, 0xc0530470e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc053047080}, 0xedb7a834d, 0xc078069b00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc053047080}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04530ec00}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc04a60eef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc04530ec00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811140 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc029af6450}, {0xc07d2ac700, 0x1, 0xc052839218}, {0x1, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntent(0x0, {0x63c85b8, 0xc029af6450}, {{{0xc08c8150b0, 0x27, 0x30}, {0x0, 0x0, 0x0}}, {{0xb, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:841 +0x9b github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc02dfdd720, {0x63c85b8, 0xc029af6450}, {0xc02aa77200, {0x1743011072cc8278, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:549 +0x6db github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc029af6450}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc02dfdd720, {0x63c85b8, 0xc029af6450}, {0xc02aa77200, {0x1743011072cc8278, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd6d0, {0x63c85b8, 0xc029af6450}, 0xc0bcee0140, 0xc0054c3558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc029af6450}, 0xc0bcee0140, {0xc02aa77200, {0x1743011072cc8278, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc029af6450}, 0xc072b38c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011072cc8278, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011072cc8278, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029af63c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029af63c0}, {0xc0054c53f8, 0x660089}, 0xc0054c5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029af63c0}, {0x7fe34e3f1c58}, 0xc072b38a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029af6330}, 0xc072b38a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}, 0xc01ed822a0, 0xc070823800) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}, 0x3, 0xc047938080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0054c5980}}}, {0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}, 0xc01ed822a0, 0xc047938080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029af6330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029af6330}, {0x40, 0xc0054c5b70}, 0xc0054c5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029af6330}, {0x4e9d7a0, 0xc072b38a00}, 0xc01ed822a0, 0xc070823800) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029af6330}, 0xc011f7a5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02aa76b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02aa76b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794598 [runnable]: internal/poll.runtime_pollWait(0x7fde186b2b80, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc062fcca80, 0xc00611f000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc062fcca80, {0xc00611f000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc062fcca80, {0xc00611f000, 0x735, 0xc03c94c8c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0394f55a0, {0xc00611f000, 0x7eda79, 0xc00611f005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01491fe00, {0xc00611f000, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0408ced38, {0xc00611f000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc09a9ea978, {0x631b900, 0xc0408ced38}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc09a9ea700, {0x7fe42ab59310, 0xc01491fe00}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc09a9ea700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc09a9ea700, {0xc011200000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc082db6ba0, {0xc011200000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc00ae1eb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc00ae1eb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00ae1f0e0, {0x634aac0, 0xc00ae1eb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00ae1ea80, 0xc088259800, 0xc07a5ad42b, 0xc07a5ad42d, {0x635bec0, 0xc02676a510}, 0xc000d6ae00, 0xc049d7be78, 0xc07a5ad429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00ae1ea80, {0x63c85b8, 0xc08f7caab0}, 0xc0882597f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09d96f036, 0x8}}, 0x0, 0xc08f7caa50, 0xc08f7caa80, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc08f7caab0}, {0x643d810, 0xc01491fe00}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc08591c570}, {0x643d810, 0xc01491fe00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc08591c570}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799748 [runnable]: sync.runtime_SemacquireMutex(0xc094eea690, 0x8f, 0x7fe15b918d40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc094eea538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc094eea560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc06d481680, {0xffffffffffffff15, 0xc094eea6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc06d481680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc06d481680, {0xc06e84ff40, 0x32, 0xa0}, {0xc02e512000, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0bd71a398, {0xc06e84ff40, 0x32, 0xa0}, {0xc02e512000, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0bd718cc0, {0xc02e512000, 0x74, 0xa0}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0bd718cc0, {0xc06e84ff40, 0x74, 0xc02e512000}, {0xc02e512000, 0xc07cedf880, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0bd718a00, {0xc02e512000, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0601e3428, {{0xc07cedf880, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc031fa1200, {{0xc02cd98540, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc091892800, {0x63c85b8, 0xc047f184b0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc047f184b0}, {0x64a6560, 0xc031fa1200}, {0xc02cd98540, 0x29, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc047f184b0}, {0x6466ed8, 0xc0601e3180}, {0xc02cd98540, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9a210}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03d195e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0205ce800, {0x63c85b8, 0xc047f184b0}, 0xc03d195e00, 0xc066747540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205ce800, {0x63c85b8, 0xc047f184b0}, 0xc03d195e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047f18240}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047f18240}, {0xc0933e88f0, 0x660089}, 0xc0933e88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047f18240}, {0xc00102e4e0}, 0xc03d195d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047f181e0}, 0xc03d195d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc09ed74180}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807454 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc07759d0b0}, {0xc0762fb600, 0x2, 0xc097202658}, {0x0, {0x1743010fec9af224, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc07759d0b0}, 0x12c89, 0xc01adf79e0, 0x71, 0xc034d91b20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc07759d0b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812200 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc02b944a80}, 0xc08b9ff8c0, 0x1, 0xc0352690a8, 0x519bbc, 0xc0688becc0, 0xc020c17140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02b944a80}, 0x120, 0xc035269230, 0x688bec60, 0x0, 0x660069, 0xc0688becc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc02b944a80}, 0xc0688bec60, {{{{0x0, 0x0}, {0xc06f5fd200, 0x34}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02b944a80}, 0x6352410, 0x420129e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc02b944a80}, {0xc0688beb40, {0x174301107ea5be60, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc02b944a80}, 0xc07ba45400, 0xc0352699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02b944a80}, 0x660089, {0xc0688beb40, {0x174301107ea5be60, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc02b944a80}, 0xc071d61800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107ea5be60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107ea5be60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b9449c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b9449c0}, {0xc03526b3f8, 0x660089}, 0xc03526b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b9449c0}, {0x0}, 0xc071d61600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02b944900}, 0xc071d61600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}, 0xc04c7f1ec0, 0xc05940e0c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}, 0x3, 0xc038600a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03526b980}}}, {0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}, 0xc04c7f1ec0, 0xc038600a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02b944900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b944900}, {0x40, 0xc03526bb70}, 0xc03526bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02b944900}, {0x4e9d7a0, 0xc071d61600}, 0xc04c7f1ec0, 0xc05940e0c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02b944900}, 0xc07419d7a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0688bea20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0688bea20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808101 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01192e500, {0x63c85b8, 0xc04bcf5ef0}, 0xc0278fdbc0, 0x1, 0xc0052c7ff0, 0x519bbc, 0xc07571e720, 0xc089ce5ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04bcf5ef0}, 0x120, 0xc0052c8178, 0x7571e6c0, 0x0, 0x660069, 0xc07571e720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01192e500, {0x63c85b8, 0xc04bcf5ef0}, 0xc07571e6c0, {{{{0x0, 0x0}, {0xc019736a20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04bcf5ef0}, 0x6352410, 0x13b806c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc007eead70, {0x63c85b8, 0xc04bcf5ef0}, {0xc07571e5a0, {0x1743011057a24815, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc007eead70, {0x63c85b8, 0xc04bcf5ef0}, 0xc02a2d5220, 0xc076022938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04bcf5ef0}, 0x660089, {0xc07571e5a0, {0x1743011057a24815, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000816800, {0x63c85b8, 0xc04bcf5ef0}, 0xc097ec5800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bcf5e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bcf5e60}, {0xc076024340, 0x660089}, 0xc076024348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bcf5e60}, {0xc00102e4e0}, 0xc097ec5500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bcf5da0}, 0xc097ec5500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc054c16f30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806758 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc08e3fb740}, 0xc01f9b6eb8, 0x1, 0xc0030830a8, 0x519bbc, 0xc01542ecc0, 0xc04a16f6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08e3fb740}, 0x120, 0xc003083230, 0x1542ec60, 0x0, 0x660069, 0xc01542ecc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc08e3fb740}, 0xc01542ec60, {{{{0x0, 0x0}, {0xc093ef9680, 0x111}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08e3fb740}, 0x6352410, 0x23d18a78) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc08e3fb740}, {0xc01542eb40, {0x1743011050c4e8ac, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc08e3fb740}, 0xc075327220, 0xc0030839f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08e3fb740}, 0x660089, {0xc01542eb40, {0x1743011050c4e8ac, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc08e3fb740}, 0xc097d1d000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050c4e8ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050c4e8ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08e3fb6b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08e3fb6b0}, {0xc0030853f8, 0x660089}, 0xc003085400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08e3fb6b0}, {0x0}, 0xc097d1ce00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08e3fb620}, 0xc097d1ce00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}, 0xc09069cfc0, 0xc0226700a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}, 0x3, 0xc08d700c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003085980}}}, {0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}, 0xc09069cfc0, 0xc08d700c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08e3fb620}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08e3fb620}, {0x40, 0xc003085b70}, 0xc003085ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08e3fb620}, {0x4e9d7a0, 0xc097d1ce00}, 0xc09069cfc0, 0xc0226700a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08e3fb620}, 0xc0ce2b8660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01542e6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01542e6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219428118 [runnable]: internal/poll.runtime_pollWait(0x7fdc6af29ca8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc08a97be80, 0xc03e3d8800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc08a97be80, {0xc03e3d8800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc08a97be80, {0xc03e3d8800, 0x735, 0xc03600e8c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0231aca28, {0xc03e3d8800, 0x7eda79, 0xc03e3d8805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc029b6afc0, {0xc03e3d8800, 0x35318b4, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc027beb620, {0xc03e3d8800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc02848c278, {0x631b900, 0xc027beb620}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc02848c000, {0x7fe42ab59310, 0xc029b6afc0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc02848c000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc02848c000, {0xc037ef5000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0431db518, {0xc037ef5000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc035fd6088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc035fd6088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc035fd6660, {0x634aac0, 0xc035fd6088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc035fd6000, 0xc08fbe2580, 0xc07d67542b, 0xc07d67542d, {0x635bec0, 0xc05a273680}, 0xc000d6ae00, 0xc046d981d8, 0xc07d675429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc035fd6000, {0x63c85b8, 0xc05c7d8d20}, 0xc08fbe2570, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0749b706d, 0x8}}, 0x0, 0xc05c7d8c90, 0xc05c7d8cf0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc05c7d8d20}, {0x643d810, 0xc029b6afc0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0719f83f0}, {0x643d810, 0xc029b6afc0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0719f83f0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810730 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ffd52c0, {0x63c85b8, 0xc06b0efce0}, 0xc04ebf7548, 0x1, 0xc00da230a8, 0x519bbc, 0xc04e7fc780, 0xc057b89980) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0731b4900, {0x63c85b8, 0xc06b0efce0}, 0xc04e7fc780, 0xc00da231a8, 0x44, 0x60, 0x0, 0xc04e7fc780, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ffd52c0, {0x63c85b8, 0xc06b0efce0}, 0xc04e7fc6c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc03b87db00, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06b0efce0}, 0x6352260, 0x7818ee68) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01f9d9590, {0x63c85b8, 0xc06b0efce0}, {0xc04e7fc5a0, {0x1743011032101887, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01f9d9590, {0x63c85b8, 0xc06b0efce0}, 0xc05292f860, 0xc00da239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b0efce0}, 0x660089, {0xc04e7fc5a0, {0x1743011032101887, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020010800, {0x63c85b8, 0xc06b0efce0}, 0xc08cb12c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032101887, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032101887, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b0efc50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b0efc50}, {0xc00da253f8, 0x660089}, 0xc00da25400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b0efc50}, {0x7fe3b8a5bac0}, 0xc08cb12a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b0efbc0}, 0xc08cb12a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}, 0xc03e2f4460, 0xc031aa0a38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}, 0x3, 0xc0768871c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00da25980}}}, {0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}, 0xc03e2f4460, 0xc0768871c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b0efbc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b0efbc0}, {0x40, 0xc00da25b70}, 0xc00da25ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b0efbc0}, {0x4e9d7a0, 0xc08cb12a00}, 0xc03e2f4460, 0xc031aa0a38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b0efbc0}, 0xc01c45b380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc072023b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc072023b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809247 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0112cdf80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812785 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807309 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02532b400, {0x63c85b8, 0xc05289b980}, 0xc08e249a00, 0xc05f31c640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc05289b980}, 0xc08e249a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057003c10, 0x0, 0x0}, 0xc08c6a3810, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057003c10, 0x0, 0x0}, 0xc08c6a3810, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05289b8f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05289b8f0}, {0xc0056093f8, 0x660089}, 0xc005609400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05289b8f0}, {0x3feffe1328e8ab1a}, 0xc08e249600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05289b860}, 0xc08e249600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}, 0xc07e300820, 0xc042adcbe8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}, 0x3, 0xc05b8d0800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005609980}}}, {0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}, 0xc07e300820, 0xc05b8d0800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05289b860}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05289b860}, {0x40, 0xc005609b70}, 0xc005609ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05289b860}, {0x4e9d7a0, 0xc08e249600}, 0xc07e300820, 0xc042adcbe8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05289b860}, 0xc045ec8f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07b8f98c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07b8f98c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810277 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc025dbbc00, {0x63c85b8, 0xc06f084ff0}, 0xc059955800, 0xc0440954a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025dbbc00, {0x63c85b8, 0xc06f084ff0}, 0xc059955800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c829ae6, 0x0, 0x0}, 0xc03cc76c30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c829ae6, 0x0, 0x0}, 0xc03cc76c30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f084f60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f084f60}, {0xc026fdb3f8, 0x660089}, 0xc026fdb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f084f60}, {0xc0015221e0}, 0xc059955600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f084ed0}, 0xc059955600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}, 0xc00f3f51e0, 0xc03135af90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}, 0x3, 0xc063d6bac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026fdb980}}}, {0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}, 0xc00f3f51e0, 0xc063d6bac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f084ed0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f084ed0}, {0x40, 0xc026fdbb70}, 0xc026fdbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f084ed0}, {0x4e9d7a0, 0xc059955600}, 0xc00f3f51e0, 0xc03135af90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f084ed0}, 0xc0bb53e9c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0638b05a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0638b05a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219792444 [runnable]: sync.runtime_SemacquireMutex(0xc003999198, 0x8f, 0x7fda7af6e740) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc003999040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc003999068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc01147db00, {0xffffffffffffff15, 0xc0039991c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc01147db00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc01147db00, {0xc0183f5c00, 0x32, 0x6f}, {0xc0022fc300, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093199998, {0xc0183f5c00, 0x32, 0x6f}, {0xc0022fc300, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0931982c0, {0xc0022fc300, 0x79, 0x6f}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0931982c0, {0xc0183f5c00, 0x79, 0xc0022fc300}, {0xc0022fc300, 0xc011a22870, 0x31}, 0x85) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093198000, {0xc0022fc300, 0x32, 0x79}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06c0358a8, {{0xc011a22870, 0x31, 0x85}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04ce5d000, {{0xc022cc8720, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc049624800, {0x63c85b8, 0xc0ab06d8f0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0ab06d8f0}, {0x64a6560, 0xc04ce5d000}, {0xc022cc8720, 0x3fefffff78658355, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0ab06d8f0}, {0x6466ed8, 0xc06c035600}, {0xc022cc8720, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07c51a8e8}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04de36700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209bc400, {0x63c85b8, 0xc0ab06d8f0}, 0xc04de36700, 0xc050935400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209bc400, {0x63c85b8, 0xc0ab06d8f0}, 0xc04de36700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0ab06d860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ab06d860}, {0xc00399f3f8, 0x660089}, 0xc00399f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0ab06d860}, {0x40d1c2bfb9a53851}, 0xc04de36400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0ab06d7d0}, 0xc04de36400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}, 0xc03ebdc4a0, 0xc0254ed860) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}, 0x3, 0xc0553f5b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00399f980}}}, {0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}, 0xc03ebdc4a0, 0xc0553f5b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0ab06d7d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ab06d7d0}, {0x40, 0xc00399fb70}, 0xc00399fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0ab06d7d0}, {0x4e9d7a0, 0xc04de36400}, 0xc03ebdc4a0, 0xc0254ed860) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0ab06d7d0}, 0xc07419c6c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0487a9b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0487a9b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812521 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219685955 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc040450790, 0x5f) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc035b5d500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc035b5db60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03aab2c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03aab2c00, {0x63c8510, 0xc0404507c0}, 0xc058bd3dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04bbe7920) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0404507c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04bbe7920) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810045 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc0612342d0}, 0xc0ac873300, 0xc06807a320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0612342d0}, 0xc0ac873300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bcee233, 0x0, 0x0}, 0xc03cc76620, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bcee233, 0x0, 0x0}, 0xc03cc76620, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc061234240}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc061234240}, {0xc0766b7a08, 0x660089}, 0xc0766b7a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc061234240}, {0xc00102e4e0}, 0xc0ac873200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0612341e0}, 0xc0ac873200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc061234150}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc061234150}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803733 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc077722c90}, 0xc053fc5a88, 0x1, 0xc0758570a8, 0x519bbc, 0xc0475158c0, 0xc048d147e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc077722c90}, 0x1e0, 0xc075857230, 0x47515860, 0x0, 0x660069, 0xc0475158c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc077722c90}, 0xc047515860, {{{{0x0, 0x0}, {0xc04d700b40, 0xe2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc077722c90}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc077722c90}, {0xc041e039e0, {0x174301102ce86235, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc077722c90}, 0xc04cee88c0, 0xc0758579f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077722c90}, 0x660089, {0xc041e039e0, {0x174301102ce86235, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc077722c90}, 0xc04366ec00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ce86235, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ce86235, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077722c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077722c00}, {0xc0758593f8, 0x660089}, 0xc075859400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077722c00}, {0x7fe3f1f91858}, 0xc04366ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077722b70}, 0xc04366ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}, 0xc0334e2760, 0xc04d5cdaa0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}, 0x3, 0xc09ce68d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075859980}}}, {0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}, 0xc0334e2760, 0xc09ce68d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc077722b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077722b70}, {0x40, 0xc075859b70}, 0xc075859ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc077722b70}, {0x4e9d7a0, 0xc04366ea00}, 0xc0334e2760, 0xc04d5cdaa0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc077722b70}, 0xc03a6b9ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc041e038c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc041e038c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808118 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0979bea80}, 0xc03d86a600, 0xc031691900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0979bea80}, 0xc03d86a600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b266044, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b266044, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0979be9f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0979be9f0}, {0xc006fe33f8, 0x660089}, 0xc006fe3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0979be9f0}, {0x3fefffdc5f52ef1d}, 0xc03d86a300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0979be960}, 0xc03d86a300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}, 0xc058ddbac0, 0xc029bccc90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}, 0x3, 0xc01bedc5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006fe3980}}}, {0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}, 0xc058ddbac0, 0xc01bedc5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0979be960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0979be960}, {0x40, 0xc006fe3b70}, 0xc006fe3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0979be960}, {0x4e9d7a0, 0xc03d86a300}, 0xc058ddbac0, 0xc029bccc90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0979be960}, 0xc041bee000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0648a47e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0648a47e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801607 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc07028f860}, 0xc0269c52d8, 0x1, 0xc051cf90a8, 0x519bbc, 0xc033c21c80, 0xc033e4e9c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07028f860}, 0x1e0, 0xc051cf9230, 0x33c21c20, 0x0, 0x660069, 0xc033c21c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc07028f860}, 0xc033c21c20, {{{{0x0, 0x0}, {0xc05f5ee480, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07028f860}, 0x63521a0, 0x65bed508) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc07028f860}, {0xc022d64900, {0x174301101aa4b463, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc07028f860}, 0xc059941c20, 0xc051cf99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07028f860}, 0x660089, {0xc022d64900, {0x174301101aa4b463, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07028f860}, 0xc0a1a6f200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101aa4b463, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101aa4b463, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07028f7d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07028f7d0}, {0xc051cfb3f8, 0x660089}, 0xc051cfb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07028f7d0}, {0x7fe426bcf5e8}, 0xc0a1a6f000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07028f740}, 0xc0a1a6f000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}, 0xc0569ee5c0, 0xc0205b1bd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}, 0x3, 0xc04627a180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc051cfb980}}}, {0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}, 0xc0569ee5c0, 0xc04627a180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07028f740}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07028f740}, {0x40, 0xc051cfbb70}, 0xc051cfbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07028f740}, {0x4e9d7a0, 0xc0a1a6f000}, 0xc0569ee5c0, 0xc0205b1bd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07028f740}, 0xc06eed4ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0ab360360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0ab360360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812718 [runnable]: sync.runtime_SemacquireMutex(0xc03fefee28, 0xcd, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x459e98, 0xc00074a680, 0xc03fefee78) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x118663c, 0xc03fefeed0, 0xc03fefeeb0, 0x4bb045) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7248b00, 0x2da4}, 0x0, 0xc0bf312ff8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0bf312d80, {0xffffffffffffff15, 0xc03feff040}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0bf312d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0bf312d80, {0xc018422fc0, 0x2e, 0x52}, 0x13, 0x13) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekGE(0xc0bf312d80, {0xc018422fc0, 0x2e, 0x52}, 0xe0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:513 +0xaf github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekGE(0xc0bf312d80, {0xc018422fc0, 0x2e, 0x52}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1317 +0x29d github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc0860d4898, {0xc018422fc0, 0x2e, 0x52}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:409 +0xd6 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0860d31c0, {0xc018422fc0, 0x60, 0x52}, 0x2e, 0x2e) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc0860d31c0, {0xc018422fc0, 0x2e, 0xc02b3cf4a0}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc0860d2f00, {0xc018422fc0, 0x2e, 0x52}, {0xc00cb44000, 0x30, 0xa0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0428faa78, {{0xc08ac155e0, 0x2d, 0xe0}, {0x0, 0x0, 0x0}}, {0xc00cb44000, 0x2f, 0xa0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:334 +0x173 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07b49c900, {{0xc076005890, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc04f93ac00, {{0xc076005890, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc04f93ac00, {0x63c85b8, 0xc050520930}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc0760058c0, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.MVCCScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc0760058c0, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2504 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc049c94630}, {{0x174301108ff2e859, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:62 +0x41b github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301108ff2e859, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0711f0400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110adc04d59, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e068000, {0x63c85b8, 0xc050520930}, 0xc0711f0400, 0xc08f98ac80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc050520930}, 0xc0711f0400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108ff2e859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108ff2e859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0505208a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0505208a0}, {0xc03ff053f8, 0x660089}, 0xc03ff05400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0505208a0}, {0x6166756e616d6e65}, 0xc0711f0200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050520810}, 0xc0711f0200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}, 0xc00abd3040, 0xc017daec60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}, 0x3, 0xc06b3d6940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03ff05980}}}, {0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}, 0xc00abd3040, 0xc06b3d6940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc050520810}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050520810}, {0x40, 0xc03ff05b70}, 0xc03ff05ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc050520810}, {0x4e9d7a0, 0xc0711f0200}, 0xc00abd3040, 0xc017daec60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc050520810}, 0xc0289e7da0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a4b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a4b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 202012769 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc084a807e0, 0xc019327440, {0x643dbd8, 0xc030a51440}, 0x80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc037a8f4d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805487 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8e000, {0x63c85b8, 0xc083524f60}, 0xc026b79b30, 0x1, 0x708017e49c2e67, 0x37e11d600, 0xc0939854a0, 0xc05fa6e720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc083524f60}, 0x120, 0xc004101230, 0x93985440, 0x0, 0x660069, 0xc0939854a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8e000, {0x63c85b8, 0xc083524f60}, 0xc093985440, {{{{0x0, 0x0}, {0xc03caeed80, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc083524f60}, 0x6352410, 0x32c541a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fdbfd60, {0x63c85b8, 0xc083524f60}, {0xc093985320, {0x17430110449da1e7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fdbfd60, {0x63c85b8, 0xc083524f60}, 0xc0831fe5a0, 0xc0041019f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc083524f60}, 0x660089, {0xc093985320, {0x17430110449da1e7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fe85000, {0x63c85b8, 0xc083524f60}, 0xc04f744400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110449da1e7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110449da1e7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc083524ed0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc083524ed0}, {0xc0041033f8, 0x660089}, 0xc004103400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc083524ed0}, {0x0}, 0xc04f744100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc083524e40}, 0xc04f744100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}, 0xc06f2378a0, 0xc026b79ae8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}, 0x3, 0xc06e0fe480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004103980}}}, {0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}, 0xc06f2378a0, 0xc06e0fe480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc083524e40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc083524e40}, {0x40, 0xc004103b70}, 0xc004103ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc083524e40}, {0x4e9d7a0, 0xc04f744100}, 0xc06f2378a0, 0xc026b79ae8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc083524e40}, 0xc044d345a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093ad3c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093ad3c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803520 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc02debaf90}, 0xc075e3ef00, 0xc01ea021e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02debaf90}, 0xc075e3ef00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102f46acf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102f46acf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02debadb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02debadb0}, {0xc0906c47f8, 0x660089}, 0xc0906c4800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02debadb0}, {0xc00102e4e0}, 0xc075e3ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02debad50}, 0xc075e3ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc06dee6e70, {0x63c85b8, 0xc02deba960}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc003255300, {0x63c85b8, 0xc02deba960}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc003255300, {0x63c8510, 0xc018051840}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc003255300, {0x63c8510, 0xc018051840}, {0x63f7db8, 0x9bf6f48}, 0xc0906c9088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc003255300, {0x63c85b8, 0xc02f01c930}, {{0x63f7db8, 0x9bf6f48}, {0xc08cdb05bd, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc003255300, {0x63c85b8, 0xc02f01c930}, {{0x63f7db8, 0x9bf6f48}, {0xc08cdb05bd, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc08cdb05bd, 0x6}, 0x0, 0x0}, {0x147c1ef2, 0xedb7a834f, 0x0}, {0x147c2246, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003255300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003255300, {0x63c8510, 0xc08689c9c0}, 0xc070133dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0671155d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc08689c9c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0671155d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808142 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e04d400, {0x63c85b8, 0xc05876f8f0}, 0xc0b3b9c400, 0xc0b7068aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc05876f8f0}, 0xc0b3b9c400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aa31fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aa31fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05876f860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05876f860}, {0xc005eb53f8, 0x660089}, 0xc005eb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05876f860}, {0x4044fffcb3216cf9}, 0xc0b3b9c200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05876f7a0}, 0xc0b3b9c200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}, 0xc071b57780, 0xc07bc921b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}, 0x3, 0xc074927f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005eb5980}}}, {0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}, 0xc071b57780, 0xc074927f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05876f7a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05876f7a0}, {0x40, 0xc005eb5b70}, 0xc005eb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05876f7a0}, {0x4e9d7a0, 0xc0b3b9c200}, 0xc071b57780, 0xc07bc921b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05876f7a0}, 0xc03a80aa20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc042cc5560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc042cc5560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808270 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02561b2c0, {0x63c85b8, 0xc059f82b10}, 0xc022316978, 0x1, 0x0, 0x1, 0xc061fcd500, 0xc03f9789c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc059f82b10}, 0x1e0, 0xc008576728, 0x61fcd4a0, 0x0, 0x660069, 0xc061fcd500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02561b2c0, {0x63c85b8, 0xc059f82b10}, 0xc061fcd4a0, {{{{0x0, 0x0}, {0xc060fc78c0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc059f82b10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc025676050, {0x63c85b8, 0xc059f82b10}, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc025676050, {0x63c85b8, 0xc059f82b10}, 0xc074aeaf00, 0xc07c606ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc059f82b10}, 0x660089, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025667000, {0x63c85b8, 0xc059f82b10}, 0xc0452da900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059f82a80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059f82a80}, {0xc07d2508f0, 0x660089}, 0xc07d2508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059f82a80}, {0xc00102e4e0}, 0xc0452da800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc059f82a20}, 0xc0452da800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc045e73ad0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811299 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc02677a8d0}, 0xc07b35db78, 0x1, 0xc0033cd0a8, 0x519bbc, 0xc06e314de0, 0xc01511ecc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02677a8d0}, 0x120, 0xc0033cd230, 0x6e314d80, 0x0, 0x660069, 0xc06e314de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc02677a8d0}, 0xc06e314d80, {{{{0x0, 0x0}, {0xc060ed8000, 0x10e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02677a8d0}, 0x6352410, 0x55bf2fd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc02677a8d0}, {0xc06e314c60, {0x17430110743b429e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc02677a8d0}, 0xc07068c140, 0xc0033cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02677a8d0}, 0x660089, {0xc06e314c60, {0x17430110743b429e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc02677a8d0}, 0xc0be438200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110743b429e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110743b429e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02677a840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02677a840}, {0xc0033cf3f8, 0x660089}, 0xc0033cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02677a840}, {0xc0015221e0}, 0xc0be438000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02677a7b0}, 0xc0be438000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}, 0xc0409df2e0, 0xc07b35dc98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}, 0x3, 0xc0473767c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0033cf980}}}, {0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}, 0xc0409df2e0, 0xc0473767c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02677a7b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02677a7b0}, {0x40, 0xc0033cfb70}, 0xc0033cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02677a7b0}, {0x4e9d7a0, 0xc0be438000}, 0xc0409df2e0, 0xc07b35dc98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02677a7b0}, 0xc04c155920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc049384d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc049384d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800771 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0966ee548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0966ee5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07bbc4cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc059745500, {0xc022c7f900, 0x2b, 0x74}, {0xc03eec9810, 0x2b, 0x6d}, 0x1, 0x4c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07bbc4b88, {0xc022c7f900, 0x1, 0x6d}, {0xc03eec9810, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07bbc4b88, {0xc022c7f900, 0x2b, 0x74}, {0xc03eec9810, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07bbc42c0, {0xc03eec9810, 0x6d, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07bbc42c0, {0xc022c7f900, 0x2b, 0xc03eec9810}, {0xc03eec9810, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07bbc4000, {0xc03eec9810, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0955d2108, {{0xc02ec84a80, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc044a22e00, {{0xc02ec84a80, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08ba7d000, {0x63c85b8, 0xc02705e270}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02705e270}, {0x64a6560, 0xc044a22e00}, {0xc02ec84a80, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02705e270}, {0x6466ed8, 0xc0955d2100}, {0xc02ec84a80, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc048db1ab8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06d04e500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b12800, {0x63c85b8, 0xc02705e270}, 0xc06d04e500, 0xc075fe1f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b12800, {0x63c85b8, 0xc02705e270}, 0xc06d04e500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bd09fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bd09fb0}, {0xc0562c28f0, 0x660089}, 0xc0562c28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bd09fb0}, {0xc00102e4e0}, 0xc06d04e400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bd09f50}, 0xc06d04e400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02aa62120}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808507 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc001639900, {0x63c85b8, 0xc0571c10e0}, 0xc04bf53590, 0x1, 0xc00dbc50a8, 0x519bbc, 0xc07af68840, 0xc07bbe5920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0571c10e0}, 0x120, 0xc00dbc5230, 0x7af687e0, 0x0, 0x660069, 0xc07af68840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc001639900, {0x63c85b8, 0xc0571c10e0}, 0xc07af687e0, {{{{0x0, 0x0}, {0xc073b63560, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0571c10e0}, 0x6352410, 0x13b81860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00aa54af0, {0x63c85b8, 0xc0571c10e0}, {0xc07af685a0, {0x174301105b77d610, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00aa54af0, {0x63c85b8, 0xc0571c10e0}, 0xc0b68cac80, 0xc00dbc59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0571c10e0}, 0x660089, {0xc07af685a0, {0x174301105b77d610, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001ae9000, {0x63c85b8, 0xc0571c10e0}, 0xc05bc3f100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0571c0ff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0571c0ff0}, {0xc00dbc73f8, 0x660089}, 0xc00dbc7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0571c0ff0}, {0x7fe429f3b1c0}, 0xc05bc3ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0571c0f00}, 0xc05bc3ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}, 0xc0476473c0, 0xc07e117428) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}, 0x3, 0xc01a45c780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00dbc7980}}}, {0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}, 0xc0476473c0, 0xc01a45c780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0571c0f00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0571c0f00}, {0x40, 0xc00dbc7b70}, 0xc00dbc7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0571c0f00}, {0x4e9d7a0, 0xc05bc3ef00}, 0xc0476473c0, 0xc07e117428) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0571c0f00}, 0xc038a639e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07af68480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07af68480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812790 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809444 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02aebc2a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812743 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803711 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc09cb83ce0}, 0xc03ef989c0, 0x1, 0xc05fd6f0a8, 0x519bbc, 0xc04c0d0ba0, 0xc04b266d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc09cb83ce0}, 0x1e0, 0xc05fd6f230, 0x4c0d0b40, 0x0, 0x660069, 0xc04c0d0ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc09cb83ce0}, 0xc04c0d0b40, {{{{0x0, 0x0}, {0xc035998000, 0xe8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc09cb83ce0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc09cb83ce0}, {0xc04d338a20, {0x1743011030de6288, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc09cb83ce0}, 0xc06d642780, 0xc05fd6f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc09cb83ce0}, 0x660089, {0xc04d338a20, {0x1743011030de6288, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc09cb83ce0}, 0xc0613c3b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011030de6288, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011030de6288, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09cb83c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09cb83c50}, {0xc05fd713f8, 0x660089}, 0xc05fd71400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09cb83c50}, {0x7fe3ac137938}, 0xc0613c3900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09cb83b90}, 0xc0613c3900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}, 0xc082addc80, 0xc038312cf0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}, 0x3, 0xc02943b6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05fd71980}}}, {0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}, 0xc082addc80, 0xc02943b6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09cb83b90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09cb83b90}, {0x40, 0xc05fd71b70}, 0xc05fd71ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09cb83b90}, {0x4e9d7a0, 0xc0613c3900}, 0xc082addc80, 0xc038312cf0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09cb83b90}, 0xc062245c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04d338900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04d338900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219696711 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaffb3b00, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc060c78d80, 0xc09b4bb000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc060c78d80, {0xc09b4bb000, 0x2b7e, 0x2b7e}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc060c78d80, {0xc09b4bb000, 0x2b7e, 0xc052749180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc043f288f0, {0xc09b4bb000, 0x7eda79, 0xc09b4bb005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03e9bd140, {0xc09b4bb000, 0x0, 0x2b7e}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02ce9bc50, {0xc09b4bb000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc08210e5f8, {0x631b900, 0xc02ce9bc50}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc08210e380, {0x7fe42ab59310, 0xc03e9bd140}, 0x2b7e) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc08210e380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc08210e380, {0xc01a2d0000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc07820b668, {0xc01a2d0000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc040f32088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc040f32088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc040f32660, {0x634aac0, 0xc040f32088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc040f32000, 0xc027041470, 0xc01a14f42b, 0xc01a14f42d, {0x635bec0, 0xc026e2c120}, 0xc000d6ae00, 0xc0be1052d8, 0xc01a14f429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc040f32000, {0x63c85b8, 0xc068ad6ea0}, 0xc027041460, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc090251037, 0x8}}, 0x0, 0xc068ad6e40, 0xc068ad6e70, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc068ad6ea0}, {0x643d810, 0xc03e9bd140}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0639f4480}, {0x643d810, 0xc03e9bd140}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0639f4480}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219772514 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0450afb00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0202d9d90, {0x4d83cc0, 0xc07dafba80}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x11e) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0450af8c0, 0xc0724faee8, 0xc0724faeb8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0450af8c0, {0x4d83cc0, 0xc07dafba80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc040c6a6c0}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc04a4cd700}, {0x4d83cc0, 0xc07dafba80}, 0x2, {0xc00c5825c0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc040c6a6c0}, {0x4f6406d, 0x7fe457d7f5b8}, {0x4e9d7a0, 0xc04a4cd700}, {0x4d83cc0, 0xc07dafba80}, 0x5360584b5a675a0c, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x39c8e2f322c0540, {0x63c85b8, 0xc040c6a6c0}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc04a4cd700}, {0x4d83cc0, 0xc07dafba80}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc03eaa1888, {0x63c85b8, 0xc040c6a6c0}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc03eaa1888}}, {0x63c85b8, 0xc040c6a6c0}, 0xc04a4cd700, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1b, 0x1a, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc02862c4b0, {0x63c85b8, 0xc040c6a660}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc02862c4b0, {0x63c85b8, 0xc040c6a660}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc04e797860, {0x63c85b8, 0xc040c6a660}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0af5df180, {0x63c85b8, 0xc040c6a660}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc04e572460) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc07548d340) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc0302d5c20) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).next(0xc075249ce0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:374 +0x7d github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc075249ce0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:189 +0x32c github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc07548d3c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*selBoolOp).Next(0xc07548d400) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:142 +0x2d github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BoolVecToSelOp).Next(0xc04e797810) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/bool_vec_to_sel.go:62 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0cc723b00) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc03f812630, {0x63c8510, 0xc07548d300}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0791c1310, {0x63c8510, 0xc07548d300}, 0xc0202d9550) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc07548cc00}, 0xc0222c1ea0, 0xc0202d9550, 0xc07dafb800, 0xc058a28700, 0xc04b3dcfe8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc07548cc00}, 0xc04b3dccf0, 0xc0222c1ea0, 0xc046b7fbc0, {{0x63ca340, 0xc07dafb780}, 0x0}, 0xc058a28700) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc04b3dc600, {0x63c8510, 0xc07548cc00}, 0xc04b3dccf0, 0xc07548cc00, {0x7fe42b790cf0, 0xc046b7fbc0}, 0x0, 0xc0302d5af8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc04b3dc600, {0x63c8510, 0xc07548cc00}, 0xc04b3dccf0, {0x7fe42b790cf0, 0xc046b7fbc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc04b3dc600, {0x63c85b8, 0xc0440126c0}, {{0x63f8d78, 0xc079bb69b0}, {0xc08ecad0a4, 0x8c}, 0x3, 0x1}, 0xc079bc85a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc04b3dc600, {0x63c85b8, 0xc0440126c0}, {{0x63f8d78, 0xc079bb69b0}, {0xc08ecad0a4, 0x8c}, 0x3, 0x1}, 0xc079bc85a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc04b3dc600, {0x63c85b8, 0xc0440126c0}, {0xc079bc85a0, {0xc044012780, 0x3, 0x3}, {0xc04940a640, 0xe, 0xe}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc019f80d08, 0x0}, 0x0, {0x331eccc9, 0xedb7a834e, 0x0}, 0x1}, 0xc04b3dc600, 0xc072501980, 0xc072501970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04b3dc600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04b3dc600, {0x63c8510, 0xc04028df80}, 0xc06fea9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc09b369180) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc04028df80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc09b369180) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809530 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b9a2f00, {0x63c85b8, 0xc02ee2f890}, 0xc07dfbb818, 0x1, 0xc01f8090a8, 0x519bbc, 0xc088144720, 0xc0137195c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02ee2f890}, 0x120, 0xc01f809230, 0x881446c0, 0x0, 0x660069, 0xc088144720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b9a2f00, {0x63c85b8, 0xc02ee2f890}, 0xc0881446c0, {{{{0x0, 0x0}, {0xc0118c8240, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02ee2f890}, 0x6352410, 0x57085440) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0fbb30, {0x63c85b8, 0xc02ee2f890}, {0xc0881445a0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0fbb30, {0x63c85b8, 0xc02ee2f890}, 0xc057f903c0, 0xc01f8099f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02ee2f890}, 0x660089, {0xc0881445a0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9bd400, {0x63c85b8, 0xc02ee2f890}, 0xc05f0d3200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ee2f680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ee2f680}, {0xc01f80b3f8, 0x660089}, 0xc01f80b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ee2f680}, {0x0}, 0xc05f0d3000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ee2f380}, 0xc05f0d3000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}, 0xc02413d820, 0xc041f29470) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}, 0x3, 0xc075816e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f80b980}}}, {0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}, 0xc02413d820, 0xc075816e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02ee2f380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ee2f380}, {0x40, 0xc01f80bb70}, 0xc01f80bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02ee2f380}, {0x4e9d7a0, 0xc05f0d3000}, 0xc02413d820, 0xc041f29470) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02ee2f380}, 0xc06c847020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086f69560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086f69560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803333 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc03e81e030}, 0xc052075400, 0xc02e5fb720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03e81e030}, 0xc052075400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b2a2ba3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b2a2ba3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0638d1f20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0638d1f20}, {0xc06094f3f8, 0x660089}, 0xc06094f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0638d1f20}, {0x54414450555f5353}, 0xc052075000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0638d1e90}, 0xc052075000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}, 0xc029281d00, 0xc0babbfae8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}, 0x3, 0xc039ed1800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06094f980}}}, {0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}, 0xc029281d00, 0xc039ed1800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0638d1e90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0638d1e90}, {0x40, 0xc06094fb70}, 0xc06094fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0638d1e90}, {0x4e9d7a0, 0xc052075000}, 0xc029281d00, 0xc0babbfae8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0638d1e90}, 0xc0d1093860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0352606c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0352606c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810178 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc074971620}, {0xc022a5cdc0, 0x2, 0xc0110c3e58}, {0x0, {0x1743010ff51d542b, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc074971620}, 0x16936, 0xc06b744a20, 0x20, 0xc046cd5810) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc074971620}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219615065 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa0008, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc08c316800, 0xc01d018800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc08c316800, {0xc01d018800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc08c316800, {0xc01d018800, 0x735, 0xc056446c80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01b1236b8, {0xc01d018800, 0x7eda79, 0xc01d018805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06270f140, {0xc01d018800, 0xc077964c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc01ea1d140, {0xc01d018800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0cdca25f8, {0x631b900, 0xc01ea1d140}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0cdca2380, {0x7fe42ab59310, 0xc06270f140}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0cdca2380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0cdca2380, {0xc060c18000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0689f33b0, {0xc060c18000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0742ae088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0742ae088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0742ae660, {0x634aac0, 0xc0742ae088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0742ae000, 0xc016e0d8d0, 0xc07796542b, 0xc07796542d, {0x635bec0, 0xc0311e2e10}, 0xc000d6ae00, 0xc04041bd98, 0xc077965429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0742ae000, {0x63c85b8, 0xc02910b7d0}, 0xc016e0d8c0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09cb0f04a, 0x8}}, 0x0, 0xc02910b710, 0xc02910b7a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02910b7d0}, {0x643d810, 0xc06270f140}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0777cbf80}, {0x643d810, 0xc06270f140}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0777cbf80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804982 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc07b8eb410}, 0xc0483f7d28, 0x1, 0xc077e930a8, 0x519bbc, 0xc054937e60, 0xc054937c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07b8eb410}, 0x1e0, 0xc077e93230, 0x54937e00, 0xc039f1c680, 0x18, 0xc054937e60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc07b8eb410}, 0xc054937e00, {{{{0x0, 0x0}, {0xc05710c240, 0x4f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07b8eb410}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc07b8eb410}, {0xc06fced8c0, {0x174301103d252792, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc07b8eb410}, 0xc039c88e60, 0xc077e939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07b8eb410}, 0x660089, {0xc06fced8c0, {0x174301103d252792, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc07b8eb410}, 0xc011f4fc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103d252792, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103d252792, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07b8eb380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b8eb380}, {0xc077e953f8, 0x660089}, 0xc077e95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07b8eb380}, {0xc0015221e0}, 0xc011f4fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07b8eb2f0}, 0xc011f4fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}, 0xc0235c5b80, 0xc0483f7f68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}, 0x3, 0xc0443aaec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc077e95980}}}, {0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}, 0xc0235c5b80, 0xc0443aaec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07b8eb2f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b8eb2f0}, {0x40, 0xc077e95b70}, 0xc077e95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}, 0x203011, 0x203011) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07b8eb2f0}, {0x4e9d7a0, 0xc011f4fa00}, 0xc0235c5b80, 0xc0483f7f68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07b8eb2f0}, 0xc033c1f6e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06fced7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06fced7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812527 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809622 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8140, {0x63c85b8, 0xc03e7e0060}, 0xc035a43320, 0x1, 0xc01ca47ff0, 0x519bbc, 0xc07dfdf920, 0xc06d99ea80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03e7e0060}, 0x120, 0xc01ca48178, 0x7dfdf8c0, 0x0, 0x660069, 0xc07dfdf920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8140, {0x63c85b8, 0xc03e7e0060}, 0xc07dfdf8c0, {{{{0x0, 0x0}, {0xc04a04d7a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03e7e0060}, 0x6352410, 0x43391988) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bbd0, {0x63c85b8, 0xc03e7e0060}, {0xc07dfdf7a0, {0x1743011067de4198, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bbd0, {0x63c85b8, 0xc03e7e0060}, 0xc01756cc80, 0xc07cf22938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03e7e0060}, 0x660089, {0xc07dfdf7a0, {0x1743011067de4198, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214dac00, {0x63c85b8, 0xc03e7e0060}, 0xc04b0bf300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067de4198, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067de4198, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0352e5f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0352e5f80}, {0xc07cf24340, 0x660089}, 0xc07cf24348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0352e5f80}, {0xc00102e4e0}, 0xc04b0bf200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0352e5650}, 0xc04b0bf200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc034f9e360}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806188 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc064843ec0}, 0xc08d7fa198, 0x1, 0x0, 0x1, 0xc01f3d8240, 0xc0925cbc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc064843ec0}, 0x1e0, 0xc00a71e728, 0x1f3d81e0, 0x0, 0x660069, 0xc01f3d8240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc064843ec0}, 0xc01f3d81e0, {{{{0x0, 0x0}, {0xc09cbdd680, 0x5c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc064843ec0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc064843ec0}, {0xc07398c240, {0x174301104cf73160, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc064843ec0}, 0xc063e34280, 0xc082d10ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc064843ec0}, 0x660089, {0xc07398c240, {0x174301104cf73160, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc064843ec0}, 0xc06fccd700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104cf73160, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104cf73160, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064843e30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064843e30}, {0xc07d2528f0, 0x660089}, 0xc07d2528f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064843e30}, {0xc00102e4e0}, 0xc06fccd600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064843dd0}, 0xc06fccd600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc053b037a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802259 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0913f8548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0913f85b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07cd71bf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc047b41180, {0xc01c25eea0, 0x2b, 0x8a}, {0xc08fead500, 0x2b, 0x2b}, 0x1, 0x1b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07cd71a88, {0xc01c25eea0, 0x1, 0x2b}, {0xc08fead500, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07cd71a88, {0xc01c25eea0, 0x2b, 0x8a}, {0xc08fead500, 0x2b, 0x2b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07cd711c0, {0xc08fead500, 0x2b, 0x8a}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07cd711c0, {0xc01c25eea0, 0x2b, 0xc08fead500}, {0xc08fead500, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07cd70f00, {0xc08fead500, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06736db88, {{0xc052d83b60, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc062b2ad00, {{0xc052d83b60, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc06d185800, {0x63c85b8, 0xc0a8279f20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0a8279f20}, {0x64a6560, 0xc062b2ad00}, {0xc052d83b60, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0a8279f20}, {0x6466ed8, 0xc06736db80}, {0xc052d83b60, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06daaf788}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0cd6e7100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ebd400, {0x63c85b8, 0xc0a8279f20}, 0xc0cd6e7100, 0xc088618140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ebd400, {0x63c85b8, 0xc0a8279f20}, 0xc0cd6e7100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0a8279e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a8279e90}, {0xc0562c48f0, 0x660089}, 0xc0562c48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0a8279e90}, {0xc00102e4e0}, 0xc0cd6e7000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0a8279e30}, 0xc0cd6e7000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0bb639410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808379 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc01fd25080}, 0xc04b76a300, 0xc066746fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc01fd25080}, 0xc04b76a300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e0b25c3, 0x0, 0x0}, 0xc057d0de10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e0b25c3, 0x0, 0x0}, 0xc057d0de10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fd24ff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fd24ff0}, {0xc00caf53f8, 0x660089}, 0xc00caf5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fd24ff0}, {0xa5100a300a6f2a06}, 0xc04b76a000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fd24f60}, 0xc04b76a000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}, 0xc04a372d60, 0xc0258f9140) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}, 0x3, 0xc0355b78c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00caf5980}}}, {0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}, 0xc04a372d60, 0xc0355b78c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01fd24f60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fd24f60}, {0x40, 0xc00caf5b70}, 0xc00caf5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01fd24f60}, {0x4e9d7a0, 0xc04b76a000}, 0xc04a372d60, 0xc0258f9140) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01fd24f60}, 0xc052b8c180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0621a6900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0621a6900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810151 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc05ca98dc0, {0x63c85b8, 0xc017152cc0}, 0xc08a887cc8, 0x1, 0xc01b7fd0a8, 0x519bbc, 0xc0159b7800, 0xc0585f40c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0637e6360, {0x63c85b8, 0xc017152cc0}, 0xc0637e6390, 0xc01b7fd1a8, 0x50, 0x70, 0xffffffffffffffff, 0xc0159b7800, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc05ca98dc0, {0x63c85b8, 0xc017152cc0}, 0xc0159b7680, {{{{0xc064c9f680, 0x2}, {0xc075fabe60, 0x3}}, {{0xc084427c20, 0xc}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc017152cc0}, 0x63521a0, 0x51b885b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc045008410, {0x63c85b8, 0xc017152cc0}, {0xc05b8a97a0, {0x1743010feac46c21, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc045008410, {0x63c85b8, 0xc017152cc0}, 0xc075282b40, 0xc01b7fd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc017152cc0}, 0x660089, {0xc05b8a97a0, {0x1743010feac46c21, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc06ffeec00, {0x63c85b8, 0xc017152cc0}, 0xc09e22bf00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010feac46c21, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010feac46c21, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc017152a80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc017152a80}, {0xc01b7ff3f8, 0x660089}, 0xc01b7ff400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc017152a80}, {0x0}, 0xc09e22bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc017152600}, 0xc09e22bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}, 0xc03ef24540, 0xc0647f2f00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}, 0x3, 0xc060400cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01b7ff980}}}, {0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}, 0xc03ef24540, 0xc060400cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc017152600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc017152600}, {0x40, 0xc01b7ffb70}, 0xc01b7ffab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc017152600}, {0x4e9d7a0, 0xc09e22bd00}, 0xc03ef24540, 0xc0647f2f00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc017152600}, 0xc04c0cc1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05a72a900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05a72a900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806836 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b9a3900, {0x63c85b8, 0xc090065ef0}, 0xc08e6f1f98, 0x1, 0xc0034210a8, 0x519bbc, 0xc0101bdb60, 0xc00cd8a840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc090065ef0}, 0x120, 0xc003421230, 0x101bdb00, 0x0, 0x660069, 0xc0101bdb60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b9a3900, {0x63c85b8, 0xc090065ef0}, 0xc0101bdb00, {{{{0x0, 0x0}, {0xc073352ea0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc090065ef0}, 0x6352410, 0x63fdd540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b9f8050, {0x63c85b8, 0xc090065ef0}, {0xc0101bd9e0, {0x174301104de41b45, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b9f8050, {0x63c85b8, 0xc090065ef0}, 0xc08e180a00, 0xc0034219f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc090065ef0}, 0x660089, {0xc0101bd9e0, {0x174301104de41b45, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9e1400, {0x63c85b8, 0xc090065ef0}, 0xc097d4a700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104de41b45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104de41b45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc090065e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090065e60}, {0xc0034233f8, 0x660089}, 0xc003423400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc090065e60}, {0x1}, 0xc097d4a500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc090065dd0}, 0xc097d4a500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}, 0xc090080c40, 0xc05b0dc240) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}, 0x3, 0xc08ed66ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003423980}}}, {0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}, 0xc090080c40, 0xc08ed66ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc090065dd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090065dd0}, {0x40, 0xc003423b70}, 0xc003423ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc090065dd0}, {0x4e9d7a0, 0xc097d4a500}, 0xc090080c40, 0xc05b0dc240) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc090065dd0}, 0xc01e1e12c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0101bd8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0101bd8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809813 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc087449aa0}, 0xc0403c1470, 0x1, 0x0, 0x1, 0xc03edbe060, 0xc05311de60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc087449aa0}, 0x1e0, 0xc01478a728, 0x3edbe000, 0x0, 0x660069, 0xc03edbe060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc087449aa0}, 0xc03edbe000, {{{{0x0, 0x0}, {0xc071cd6240, 0x5c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc06ae79e00, {0x63c85b8, 0xc087449aa0}, 0xc05abfbab0, 0xb2c0920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc087449aa0}, {0xc085ab8d80, {0x1743011066dd663a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc087449aa0}, 0xc0225cefa0, 0xc00b2c0ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc087449aa0}, 0x660089, {0xc085ab8d80, {0x1743011066dd663a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc087449aa0}, 0xc082f58500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066dd663a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066dd663a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc087449a10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc087449a10}, {0xc002fe68f0, 0x660089}, 0xc002fe68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc087449a10}, {0xc00102e4e0}, 0xc082f58400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0874499b0}, 0xc082f58400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc034532c60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810180 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc031e75230}, 0xc02765fc80, 0x1, 0xc0048f70a8, 0x519bbc, 0xc053087f00, 0xc059ee69c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc031294360, {0x63c85b8, 0xc031e75230}, 0xc0733f1440, 0xc0048f71a8, 0x53087f00, 0x0, 0x2, 0xc053087f00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc031e75230}, 0xc0290dd080, {{{{0xc079db7b00, 0x46}, {0xc028414900, 0x5e}}, {{0xc066dd38c0, 0x90}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc031e75230}, 0x63521a0, 0x7818e4c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc031e75230}, {0xc05f38d0e0, {0x1743011002161b09, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc031e75230}, 0xc0c2a71180, 0xc0048f79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc031e75230}, 0x660089, {0xc05f38d0e0, {0x1743011002161b09, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc031e75230}, 0xc045d55f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011002161b09, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011002161b09, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031e751a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031e751a0}, {0xc0048f93f8, 0x660089}, 0xc0048f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031e751a0}, {0x4d939c}, 0xc045d55c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031e75080}, 0xc045d55c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}, 0xc07cd08f00, 0xc03c931b00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}, 0x3, 0xc04517f240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0048f9980}}}, {0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}, 0xc07cd08f00, 0xc04517f240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc031e75080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031e75080}, {0x40, 0xc0048f9b70}, 0xc0048f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc031e75080}, {0x4e9d7a0, 0xc045d55c00}, 0xc07cd08f00, 0xc03c931b00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc031e75080}, 0xc0290dcf00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05f38cfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05f38cfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812199 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02499e140, {0x63c85b8, 0xc0b9ac1350}, 0xc03b040f00, 0x1, 0xc034f070a8, 0x519bbc, 0xc00a3c4f00, 0xc047f2d140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0b9ac1350}, 0x120, 0xc034f07230, 0xa3c4ea0, 0x0, 0x660069, 0xc00a3c4f00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02499e140, {0x63c85b8, 0xc0b9ac1350}, 0xc00a3c4ea0, {{{{0x0, 0x0}, {0xc07ca007e0, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0b9ac1350}, 0x6352410, 0x47c52318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc023b3b400, {0x63c85b8, 0xc0b9ac1350}, {0xc00a3c4b40, {0x174301107ea5be60, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc023b3b400, {0x63c85b8, 0xc0b9ac1350}, 0xc066746780, 0xc034f079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0b9ac1350}, 0x660089, {0xc00a3c4b40, {0x174301107ea5be60, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0193e7c00, {0x63c85b8, 0xc0b9ac1350}, 0xc048c44800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107ea5be60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107ea5be60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0b9ac12c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b9ac12c0}, {0xc034f093f8, 0x660089}, 0xc034f09400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0b9ac12c0}, {0x7fe3c2fd21c0}, 0xc048c44600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0b9ac1230}, 0xc048c44600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}, 0xc01e5171c0, 0xc03b041ae8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}, 0x3, 0xc018743d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034f09980}}}, {0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}, 0xc01e5171c0, 0xc018743d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0b9ac1230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b9ac1230}, {0x40, 0xc034f09b70}, 0xc034f09ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0b9ac1230}, {0x4e9d7a0, 0xc048c44600}, 0xc01e5171c0, 0xc03b041ae8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0b9ac1230}, 0xc082ff9260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0688be7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0688be7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802245 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc046bf74d0}, 0xc037bcb290, 0x1, 0xc063435ff0, 0x519bbc, 0xc039579320, 0xc039c3c420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc046bf74d0}, 0x1e0, 0xc063436178, 0x395792c0, 0x0, 0x660069, 0xc039579320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc046bf74d0}, 0xc0395792c0, {{{{0x0, 0x0}, {0xc02d27cd80, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc046bf74d0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc046bf74d0}, {0xc02852e5a0, {0x17430110213c29ff, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc046bf74d0}, 0xc02fbc6c80, 0xc0768ee938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046bf74d0}, 0x660089, {0xc02852e5a0, {0x17430110213c29ff, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc046bf74d0}, 0xc04e7df100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110213c29ff, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110213c29ff, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046bf7440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046bf7440}, {0xc0768f0340, 0x660089}, 0xc0768f0348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046bf7440}, {0xc00102e4e0}, 0xc04e7df000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046bf73e0}, 0xc04e7df000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc046bf6e40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811605 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc02392ef60}, 0xc01c104c60, 0x1, 0xc0059e90a8, 0x519bbc, 0xc0179e4480, 0xc01657cd20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc02a3b03f0, {0x63c85b8, 0xc02392ef60}, 0xc067748360, 0xc0059e91a8, 0x179e4480, 0x0, 0x2, 0xc0179e4480, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc02392ef60}, 0xc0179e43c0, {{{{0xc07e013440, 0x3b}, {0xc0504e5440, 0x61}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x30, {0x63c85b8, 0xc02392ef60}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc02392ef60}, {0xc04b8e85a0, {0x174301107927c2f6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc02392ef60}, 0xc076db1f40, 0xc0059e99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02392ef60}, 0x660089, {0xc04b8e85a0, {0x174301107927c2f6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc02392ef60}, 0xc0414a8800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107927c2f6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107927c2f6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02392ee70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02392ee70}, {0xc0059eb3f8, 0x660089}, 0xc0059eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02392ee70}, {0x0}, 0xc0414a8600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02392ed20}, 0xc0414a8600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}, 0xc06ff3fc60, 0xc0554e15d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}, 0x3, 0xc06d1b8200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0059eb980}}}, {0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}, 0xc06ff3fc60, 0xc06d1b8200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02392ed20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02392ed20}, {0x40, 0xc0059ebb70}, 0xc0059ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02392ed20}, {0x4e9d7a0, 0xc0414a8600}, 0xc06ff3fc60, 0xc0554e15d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02392ed20}, 0xc02eb64b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04b8e8240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04b8e8240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804193 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6f00, {0x63c85b8, 0xc04c1ccf30}, 0xc053cdc750, 0x1, 0xc0750430a8, 0x519bbc, 0xc06101a840, 0xc04bbab5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04c1ccf30}, 0x120, 0xc075043230, 0x6101a7e0, 0x0, 0x660069, 0xc06101a840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6f00, {0x63c85b8, 0xc04c1ccf30}, 0xc06101a7e0, {{{{0x0, 0x0}, {0xc0906b9d40, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04c1ccf30}, 0x6352410, 0x41bc5c40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c22d0, {0x63c85b8, 0xc04c1ccf30}, {0xc06101a6c0, {0x17430110306734ae, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c22d0, {0x63c85b8, 0xc04c1ccf30}, 0xc04a485680, 0xc0750439f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c1ccf30}, 0x660089, {0xc06101a6c0, {0x17430110306734ae, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4000, {0x63c85b8, 0xc04c1ccf30}, 0xc0aad5bc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c1ccea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c1ccea0}, {0xc0750453f8, 0x660089}, 0xc075045400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c1ccea0}, {0xc010530970}, 0xc0aad5ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c1cce10}, 0xc0aad5ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}, 0xc03b2ff5c0, 0xc01bac27c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}, 0x3, 0xc03110d7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075045980}}}, {0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}, 0xc03b2ff5c0, 0xc03110d7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04c1cce10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c1cce10}, {0x40, 0xc075045b70}, 0xc075045ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04c1cce10}, {0x4e9d7a0, 0xc0aad5ba00}, 0xc03b2ff5c0, 0xc01bac27c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04c1cce10}, 0xc046956ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06101a5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06101a5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806937 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc06fcd3ef0}, 0xc02a6d5110, 0x1, 0xc004b730a8, 0x519bbc, 0xc0590b8ba0, 0xc097e055c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06fcd3ef0}, 0x120, 0xc004b73230, 0x590b8b40, 0x0, 0x660069, 0xc0590b8ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc06fcd3ef0}, 0xc0590b8b40, {{{{0x0, 0x0}, {0xc083859b00, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06fcd3ef0}, 0x6352410, 0x5449b628) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc06fcd3ef0}, {0xc0590b8a20, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc06fcd3ef0}, 0xc00aae8320, 0xc004b739f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06fcd3ef0}, 0x660089, {0xc0590b8a20, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc06fcd3ef0}, 0xc078eff000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06fcd3e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fcd3e60}, {0xc004b753f8, 0x660089}, 0xc004b75400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06fcd3e60}, {0x0}, 0xc078efee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fcd3dd0}, 0xc078efee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}, 0xc08fb7dce0, 0xc067797b78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}, 0x3, 0xc08efba780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004b75980}}}, {0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}, 0xc08fb7dce0, 0xc08efba780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06fcd3dd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fcd3dd0}, {0x40, 0xc004b75b70}, 0xc004b75ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06fcd3dd0}, {0x4e9d7a0, 0xc078efee00}, 0xc08fb7dce0, 0xc067797b78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06fcd3dd0}, 0xc040274f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590b87e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590b87e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805345 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc025d78140, {0x63c85b8, 0xc075141f20}, 0xc017751860, 0x1, 0xc07ba5d0a8, 0x519bbc, 0xc05b7347e0, 0xc051edad80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc075141f20}, 0x1e0, 0xc07ba5d230, 0x5b734780, 0xc039f1dba0, 0x18, 0xc05b7347e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc025d78140, {0x63c85b8, 0xc075141f20}, 0xc05b734780, {{{{0x0, 0x0}, {0xc046250d80, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc075141f20}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc025d3c550, {0x63c85b8, 0xc075141f20}, {0xc05b776480, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc025d3c550, {0x63c85b8, 0xc075141f20}, 0xc066334500, 0xc07ba5d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc075141f20}, 0x660089, {0xc05b776480, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025d5dc00, {0x63c85b8, 0xc075141f20}, 0xc065240900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075141e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075141e90}, {0xc07ba5f3f8, 0x660089}, 0xc07ba5f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075141e90}, {0x0}, 0xc065240700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc075141e00}, 0xc065240700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}, 0xc064a6da80, 0xc015d6b428) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}, 0x3, 0xc062e82240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07ba5f980}}}, {0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}, 0xc064a6da80, 0xc062e82240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc075141e00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075141e00}, {0x40, 0xc07ba5fb18}, 0xc07ba5fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}, 0x203018, 0x203018) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc075141e00}, {0x4e9d7a0, 0xc065240700}, 0xc064a6da80, 0xc015d6b428) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc075141e00}, 0xc0189bcc00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b776360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b776360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812781 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804325 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc05a6bc7b0}, 0xc00d8efe00, 0xc06289ad20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc05a6bc7b0}, 0xc00d8efe00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103208748e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103208748e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05a6bc720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05a6bc720}, {0xc079e993f8, 0x660089}, 0xc079e99400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05a6bc720}, {0x4d939c}, 0xc00d8efc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05a6bc660}, 0xc00d8efc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}, 0xc01a53e620, 0xc0117df368) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}, 0x3, 0xc03fbe7800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079e99980}}}, {0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}, 0xc01a53e620, 0xc03fbe7800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05a6bc660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05a6bc660}, {0x40, 0xc079e99b70}, 0xc079e99ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05a6bc660}, {0x4e9d7a0, 0xc00d8efc00}, 0xc01a53e620, 0xc0117df368) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05a6bc660}, 0xc049e27ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc056fdeb40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc056fdeb40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810949 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0310c9900, {0x63c85b8, 0xc0452dc0c0}, 0xc04dc8b6b0, 0x1, 0x0, 0x1, 0xc0434b4cc0, 0xc0485764e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0452dc0c0}, 0x120, 0xc010860728, 0x434b4c60, 0x0, 0x660069, 0xc0434b4cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0310c9900, {0x63c85b8, 0xc0452dc0c0}, 0xc0434b4c60, {{{{0x0, 0x0}, {0xc04e62ea20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0452dc0c0}, 0x6352410, 0x1b309a58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc03107f450, {0x63c85b8, 0xc0452dc0c0}, {0xc07a39b200, {0x1743011070203ed7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc03107f450, {0x63c85b8, 0xc0452dc0c0}, 0xc04bbd0780, 0xc06318aee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0452dc0c0}, 0x660089, {0xc07a39b200, {0x1743011070203ed7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0309e0800, {0x63c85b8, 0xc0452dc0c0}, 0xc08d91e100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011070203ed7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011070203ed7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0452dc030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0452dc030}, {0xc0ad83c8f0, 0x660089}, 0xc0ad83c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0452dc030}, {0x24}, 0xc08d91e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077345fb0}, 0xc08d91e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07c4e8630}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805448 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc041ca15c0}, 0xc0465cbe48, 0x1, 0x708017e142ce9d, 0x37e11d600, 0xc05f70ba40, 0xc06f6b8cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041ca15c0}, 0x120, 0xc0794b9230, 0x5f70b9e0, 0xc050750680, 0x18, 0xc05f70ba40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc041ca15c0}, 0xc05f70b9e0, {{{{0x0, 0x0}, {0xc05c808000, 0x22}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041ca15c0}, 0x6352410, 0x24ff1418) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc041ca15c0}, {0xc05f70b8c0, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc041ca15c0}, 0xc023ca0dc0, 0xc0794b99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041ca15c0}, 0x660089, {0xc05f70b8c0, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc041ca15c0}, 0xc06bc1e500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041ca1530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ca1530}, {0xc0794bb3f8, 0x660089}, 0xc0794bb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041ca1530}, {0x4677c5}, 0xc06bc1e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041ca14a0}, 0xc06bc1e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}, 0xc01389a3a0, 0xc0465cbe18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}, 0x3, 0xc074875dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0794bb980}}}, {0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}, 0xc01389a3a0, 0xc074875dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041ca14a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ca14a0}, {0x40, 0xc0794bbb70}, 0xc0794bbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}, 0x20301d, 0x20301d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041ca14a0}, {0x4e9d7a0, 0xc06bc1e000}, 0xc01389a3a0, 0xc0465cbe18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041ca14a0}, 0xc02cab0f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f70b200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05f70b200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803223 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04967b620}, 0xc026fa9938, 0x1, 0xc06ad2b0a8, 0x519bbc, 0xc0457e7500, 0xc03fc43aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04967b620}, 0x1e0, 0xc06ad2b230, 0x457e74a0, 0x0, 0x660069, 0xc0457e7500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04967b620}, 0xc0457e74a0, {{{{0x0, 0x0}, {0xc077c91b00, 0x3c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04967b620}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04967b620}, {0xc03487e7e0, {0x174301102b7eff25, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04967b620}, 0xc042350460, 0xc06ad2b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04967b620}, 0x660089, {0xc03487e7e0, {0x174301102b7eff25, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04967b620}, 0xc040a2b300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b7eff25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b7eff25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04967b590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04967b590}, {0xc06ad2d3f8, 0x660089}, 0xc06ad2d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04967b590}, {0x12}, 0xc040a2b100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04967b500}, 0xc040a2b100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}, 0xc0681df6e0, 0xc02733e4e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}, 0x3, 0xc01bab9400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06ad2d980}}}, {0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}, 0xc0681df6e0, 0xc01bab9400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04967b500}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04967b500}, {0x40, 0xc06ad2db70}, 0xc06ad2dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04967b500}, {0x4e9d7a0, 0xc040a2b100}, 0xc0681df6e0, 0xc02733e4e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04967b500}, 0xc01d0df3e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03487e6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03487e6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802186 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0cef7d710}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810253 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02a5f4800, {0x63c85b8, 0xc076c82960}, 0xc08ad6e000, 0xc079e328c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a5f4800, {0x63c85b8, 0xc076c82960}, 0xc08ad6e000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cabf0ca, 0x0, 0x0}, 0xc03905a5d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cabf0ca, 0x0, 0x0}, 0xc03905a5d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076c828d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076c828d0}, {0xc027e4b3f8, 0x660089}, 0xc027e4b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076c828d0}, {0x0}, 0xc0cd523e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076c82840}, 0xc0cd523e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}, 0xc07252a680, 0xc029399f20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}, 0x3, 0xc073fea0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc027e4b980}}}, {0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}, 0xc07252a680, 0xc073fea0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076c82840}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076c82840}, {0x40, 0xc027e4bb70}, 0xc027e4bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076c82840}, {0x4e9d7a0, 0xc0cd523e00}, 0xc07252a680, 0xc029399f20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076c82840}, 0xc04d6526c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0616e3b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0616e3b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807303 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0430601b0}, 0xc05708c510, 0x1, 0xc003a690a8, 0x519bbc, 0xc01b3eef60, 0xc03435b140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0430601b0}, 0x1e0, 0xc003a69230, 0x1b3eef00, 0x0, 0x660069, 0xc01b3eef60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0430601b0}, 0xc01b3eef00, {{{{0x0, 0x0}, {0xc05067d440, 0x42}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0430601b0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0430601b0}, {0xc0664bbd40, {0x1743010ff7dc5987, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0430601b0}, 0xc07471b5e0, 0xc003a699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0430601b0}, 0x660089, {0xc0664bbd40, {0x1743010ff7dc5987, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0430601b0}, 0xc08d7f3100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff7dc5987, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff7dc5987, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043060120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043060120}, {0xc003a6b3f8, 0x660089}, 0xc003a6b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043060120}, {0xc0015221e0}, 0xc08d7f2f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043060090}, 0xc08d7f2f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}, 0xc006dce2a0, 0xc05708df20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}, 0x3, 0xc00ea58c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003a6b980}}}, {0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}, 0xc006dce2a0, 0xc00ea58c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc043060090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043060090}, {0x40, 0xc003a6bb70}, 0xc003a6bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc043060090}, {0x4e9d7a0, 0xc08d7f2f00}, 0xc006dce2a0, 0xc05708df20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc043060090}, 0xc02269cf60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0664bbc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0664bbc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801490 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc078363dd0}, 0xc024f23488, 0x1, 0xc0060bbf20, 0x519bbc, 0xc042d08660, 0xc05a936540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc078363dd0}, 0xc0, 0xc0060bc0a8, 0x42d08600, 0x0, 0x660069, 0xc042d08660, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc078363dd0}, 0xc042d08600, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc078363dd0}, 0x63524a0, 0x5591d310) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc078363dd0}, {0x0, {0x1743011056bd6650, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc078363dd0}, 0xc0673414a0, 0xc0060bc868) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc078363dd0}, 0x660089, {0x0, {0x1743011056bd6650, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc078363dd0}, 0xc08ad6b700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078363d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078363d40}, {0xc0060be270, 0x660089}, 0xc0060be278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078363d40}, {0xc037d4e3f0}, 0xc08ad6b600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078363ce0}, 0xc08ad6b600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x1743011056bd6650, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc01e1ff290}, 0x0, 0xc0616c0100) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc01e1ff290}, 0x66040f) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc019919860, {0x63c85b8, 0xc01e1ff290}, {{0xa, 0x4a, 0x6c, 0xd7, 0x50, 0x7f, 0x4f, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:922 +0x1b0 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc019919860, {0x63c85b8, 0xc01e1ff290}, 0xc03411a960, 0xc070b3d6e0, 0xc04eee8c30) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:610 +0xd4a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc01e1ff290}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc037d4e510}, {{0xc0609e93c0, 0x24, 0x24}}, 0xc0060c1380) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc019919860, {0x63c85b8, 0xc037d4e510}, 0xc03411a960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0301af310, {0x63c85b8, 0xc037d4e510}, {0x0, {0x174301101c5c537c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc037d4e510}, 0xc074aea3c0, 0xc0308909d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc037d4e510}, 0xc074aea3c0, {0x0, {0x174301101c5c537c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc037d4e510}, 0xc07b11ff00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c5c537c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c5c537c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301101c5c537c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc037d4e480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037d4e480}, {0xc043cb93f8, 0x660089}, 0xc043cb9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc037d4e480}, {0x0}, 0xc07b11fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc037d4e3f0}, 0xc07b11fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}, 0xc05914f360, 0xc03bc44360) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}, 0x3, 0xc02dceef80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043cb9980}}}, {0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}, 0xc05914f360, 0xc02dceef80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc037d4e3f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037d4e3f0}, {0x40, 0xc043cb9b70}, 0xc043cb9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc037d4e3f0}, {0x4e9d7a0, 0xc07b11fd00}, 0xc05914f360, 0xc03bc44360) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc037d4e3f0}, 0xc061d89da0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05b6217a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05b6217a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812088 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc027a14120}, {0xc073484000, 0x2, 0xc0971fde58}, {0x0, {0x1743010ff139bb73, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc027a14120}, 0xfe48, 0xc0505d4120, 0x20, 0xc087229070) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc027a14120}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803772 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fc82000, {0x63c85b8, 0xc049841d40}, 0xc04ef3df00, 0xc08f98ad20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fc82000, {0x63c85b8, 0xc049841d40}, 0xc04ef3df00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103074bebc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103074bebc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc049841cb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049841cb0}, {0xc07732b3f8, 0x660089}, 0xc07732b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc049841cb0}, {0x7fe4287b4d20}, 0xc04ef3dd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc049841bc0}, 0xc04ef3dd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}, 0xc073f7f1a0, 0xc02cc7ed50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}, 0x3, 0xc03c4cf940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07732b980}}}, {0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}, 0xc073f7f1a0, 0xc03c4cf940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc049841bc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049841bc0}, {0x40, 0xc07732bb70}, 0xc07732bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc049841bc0}, {0x4e9d7a0, 0xc04ef3dd00}, 0xc073f7f1a0, 0xc02cc7ed50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc049841bc0}, 0xc07d44e840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04d808a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04d808a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809726 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0039f6900}, 0xc015826408, 0x1, 0xc0172f90a8, 0x519bbc, 0xc06d941c80, 0xc073c118c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0039f6900}, 0x1e0, 0xc0172f9230, 0x6d941c20, 0x0, 0x660069, 0xc06d941c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0039f6900}, 0xc06d941c20, {{{{0x0, 0x0}, {0xc059ed4fc0, 0xf3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0039f6900}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0039f6900}, {0xc0799bcfc0, {0x1743011067eb6432, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0039f6900}, 0xc0753df900, 0xc0172f99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0039f6900}, 0x660089, {0xc0799bcfc0, {0x1743011067eb6432, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0039f6900}, 0xc07376f500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067eb6432, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067eb6432, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0039f6690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0039f6690}, {0xc0172fb3f8, 0x660089}, 0xc0172fb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0039f6690}, {0x0}, 0xc07376f300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0039f65a0}, 0xc07376f300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}, 0xc0649b7760, 0xc076fdb428) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}, 0x3, 0xc076c48f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0172fb980}}}, {0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}, 0xc0649b7760, 0xc076c48f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0039f65a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0039f65a0}, {0x40, 0xc0172fbb70}, 0xc0172fbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0039f65a0}, {0x4e9d7a0, 0xc07376f300}, 0xc0649b7760, 0xc076fdb428) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0039f65a0}, 0xc043d6d560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0799bcea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0799bcea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802119 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021a38400, {0x63c85b8, 0xc0738533e0}, 0xc0d2801500, 0xc066746140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021a38400, {0x63c85b8, 0xc0738533e0}, 0xc0d2801500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020afec97, 0x0, 0x0}, 0xc01a092df0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020afec97, 0x0, 0x0}, 0xc01a092df0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0738532f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0738532f0}, {0xc08da97918, 0x660089}, 0xc08da97920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0738532f0}, {0xc00102e4e0}, 0xc0d2801400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073853290}, 0xc0d2801400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc03db18360}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc05454eab0}, {0xc00dfd36b0, 0x24}, 0x46c73575f, 0xc003501d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc03eaae300}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc05454eab0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219738194 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc060d4c550, 0x66) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc077aa7500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc077aa7b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0c9032c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0c9032c00, {0x63c8510, 0xc060d4c600}, 0xc06da7bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06c0d0200) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc060d4c600}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06c0d0200) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219806246 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e04d400, {0x63c85b8, 0xc04da92060}, 0xc08517f900, 0xc075b4fae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc04da92060}, 0xc08517f900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110001c2e22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110001c2e22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ff8dfb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ff8dfb0}, {0xc005ad73f8, 0x660089}, 0xc005ad7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ff8dfb0}, {0xc0015221e0}, 0xc08517f700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06ff8df20}, 0xc08517f700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}, 0xc07c5b0620, 0xc08ba3fb48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}, 0x3, 0xc06e4888c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ad7980}}}, {0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}, 0xc07c5b0620, 0xc06e4888c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06ff8df20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ff8df20}, {0x40, 0xc005ad7b70}, 0xc005ad7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06ff8df20}, {0x4e9d7a0, 0xc08517f700}, 0xc07c5b0620, 0xc08ba3fb48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06ff8df20}, 0xc052474720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc047233440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc047233440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812539 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074aa90, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00d1618c0, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc0a20c8380, 0xc00d1618c0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00d1618c0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a523c0, 0xc00b9e7198, 0xc0930ec438, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc0930ec550, 0x0, 0xc0930ec550) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0930ec3e8, 0xc0158cbe40, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0930ec3e8, {0xc079b9c5a0, 0x28, 0x44}, {0xc0b5a18a00, 0x28, 0x4b}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0930eb1c0, {0xc0b5a18a00, 0x0, 0xc00b9e74e0}, 0x121ea30, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0930eb1c0, {0xc079b9c5a0, 0x28, 0x4b}, {0xc0b5a18a00, 0x27, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0930eaf00, {0xc0b5a18a00, 0x28, 0x4b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc086ad4008, {{0xc03a992720, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07b355300, {{0xc03a992720, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc053137800, {0x63c85b8, 0xc04bf64d20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04bf64d20}, {0x64a6560, 0xc07b355300}, {0xc03a992720, 0xc0534496c0, 0x30}, {0x1743011087a4b504, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04bf64d20}, {0x6466ed8, 0xc086ad4000}, {0xc03a992720, 0x27, 0x30}, {0x1743011087a4b504, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc079b5b0c8}, {{0x1743011087a4b504, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011087a4b504, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04aa84f00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a5721a04, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02f354400, {0x63c85b8, 0xc04bf64d20}, 0xc04aa84f00, 0xc052f4dc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f354400, {0x63c85b8, 0xc04bf64d20}, 0xc04aa84f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011087a4b504, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011087a4b504, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bf64c90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf64c90}, {0xc00b9ed3f8, 0x660089}, 0xc00b9ed400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bf64c90}, {0xc0015221e0}, 0xc04aa84d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bf64c00}, 0xc04aa84d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}, 0xc0ccf7f6c0, 0xc0381615d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}, 0x3, 0xc02a52ebc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b9ed980}}}, {0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}, 0xc0ccf7f6c0, 0xc02a52ebc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bf64c00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf64c00}, {0x40, 0xc00b9edb70}, 0xc00b9edab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bf64c00}, {0x4e9d7a0, 0xc04aa84d00}, 0xc0ccf7f6c0, 0xc0381615d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bf64c00}, 0xc05c2fa8a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdd8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdd8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800075 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc079be78f0}, 0xc0488ac900, 0xc045655540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc079be78f0}, 0xc0488ac900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101349e060, 0x0, 0x0}, 0xc0318376d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101349e060, 0x0, 0x0}, 0xc0318376d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079be7860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079be7860}, {0xc03a1bb3f8, 0x660089}, 0xc03a1bb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079be7860}, {0x0}, 0xc0488ac700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079be77d0}, 0xc0488ac700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}, 0xc0377c3160, 0xc0a927a768) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}, 0x3, 0xc068b15c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03a1bb980}}}, {0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}, 0xc0377c3160, 0xc068b15c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079be77d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079be77d0}, {0x40, 0xc03a1bbb70}, 0xc03a1bbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079be77d0}, {0x4e9d7a0, 0xc0488ac700}, 0xc0377c3160, 0xc0a927a768) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079be77d0}, 0xc049e99e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0120b5b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0120b5b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808492 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020f88500, {0x63c85b8, 0xc020b72f60}, 0xc0342fd1a0, 0x1, 0xc00a1690a8, 0x519bbc, 0xc05c772de0, 0xc05aa9fc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc020b72f60}, 0x120, 0xc00a169230, 0x5c772d80, 0x0, 0x660069, 0xc05c772de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020f88500, {0x63c85b8, 0xc020b72f60}, 0xc05c772d80, {{{{0x0, 0x0}, {0xc08402d440, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc020b72f60}, 0x6352410, 0x53d3d5d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc020df9ef0, {0x63c85b8, 0xc020b72f60}, {0xc05c772c60, {0x174301105d4b31f8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc020df9ef0, {0x63c85b8, 0xc020b72f60}, 0xc0b68cb720, 0xc00a1699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc020b72f60}, 0x660089, {0xc05c772c60, {0x174301105d4b31f8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210be000, {0x63c85b8, 0xc020b72f60}, 0xc066b5a600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105d4b31f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105d4b31f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020b72e70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020b72e70}, {0xc00a16b3f8, 0x660089}, 0xc00a16b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020b72e70}, {0xc06bb94c30}, 0xc066b5a400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020b72db0}, 0xc066b5a400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}, 0xc0645df820, 0xc0203272a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}, 0x3, 0xc066300e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a16b980}}}, {0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}, 0xc0645df820, 0xc066300e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020b72db0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020b72db0}, {0x40, 0xc00a16bb70}, 0xc00a16bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020b72db0}, {0x4e9d7a0, 0xc066b5a400}, 0xc0645df820, 0xc0203272a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020b72db0}, 0xc0a5939f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b280ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b280ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804873 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021cd2000, {0x63c85b8, 0xc041dc0210}, 0xc0318572f0, 0x1, 0xc08e6330a8, 0x519bbc, 0xc06ee174a0, 0xc06ed11ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041dc0210}, 0x120, 0xc08e633230, 0x6ee17440, 0xc06616d380, 0x18, 0xc06ee174a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021cd2000, {0x63c85b8, 0xc041dc0210}, 0xc06ee17440, {{{{0x0, 0x0}, {0xc038c6fe60, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041dc0210}, 0x6352410, 0x1b25d0b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c432c0, {0x63c85b8, 0xc041dc0210}, {0xc06ee17320, {0x174301103b05a390, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c432c0, {0x63c85b8, 0xc041dc0210}, 0xc0223820a0, 0xc08e6339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041dc0210}, 0x660089, {0xc06ee17320, {0x174301103b05a390, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd6000, {0x63c85b8, 0xc041dc0210}, 0xc0878b0700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b05a390, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b05a390, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041dc0180}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041dc0180}, {0xc08e6353f8, 0x660089}, 0xc08e635400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041dc0180}, {0x4677c5}, 0xc0878b0500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041dc00f0}, 0xc0878b0500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}, 0xc0235c4c00, 0xc031857b90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}, 0x3, 0xc0499ecb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc08e635980}}}, {0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}, 0xc0235c4c00, 0xc0499ecb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041dc00f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041dc00f0}, {0x40, 0xc08e635b70}, 0xc08e635ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}, 0x203012, 0x203012) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041dc00f0}, {0x4e9d7a0, 0xc0878b0500}, 0xc0235c4c00, 0xc031857b90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041dc00f0}, 0xc04816d920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06ee17200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06ee17200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810269 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bcfa000, {0x63c85b8, 0xc04fe58de0}, 0xc055577100, 0xc0842d6fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bcfa000, {0x63c85b8, 0xc04fe58de0}, 0xc055577100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cc77dd1, 0x0, 0x0}, 0xc03cc772a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cc77dd1, 0x0, 0x0}, 0xc03cc772a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04fe58d20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04fe58d20}, {0xc00b45f3f8, 0x660089}, 0xc00b45f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04fe58d20}, {0x7fe13dd08ef0}, 0xc055576e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04fe58c90}, 0xc055576e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}, 0xc00f3f5940, 0xc037c76990) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}, 0x3, 0xc057aae040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b45f980}}}, {0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}, 0xc00f3f5940, 0xc057aae040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04fe58c90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04fe58c90}, {0x40, 0xc00b45fb70}, 0xc00b45fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04fe58c90}, {0x4e9d7a0, 0xc055576e00}, 0xc00f3f5940, 0xc037c76990) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04fe58c90}, 0xc038e1e480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc017e8c120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc017e8c120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 214020868 [sync.Cond.Wait, 3 minutes]: sync.runtime_notifyListWait(0xc058f87750, 0x98) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0a377aa80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0a377b0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04a02e000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04a02e000, {0x63c8510, 0xc058f87780}, 0xc05f3bbdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02fbb37a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc058f87780}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02fbb37a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810283 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b26c000, {0x63c85b8, 0xc04440cea0}, 0xc0a3d8e900, 0xc050935c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc04440cea0}, 0xc0a3d8e900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cc9ea84, 0x0, 0x0}, 0xc03399fe10, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cc9ea84, 0x0, 0x0}, 0xc03399fe10, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04440cdb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04440cdb0}, {0xc00b45f3f8, 0x660089}, 0xc00b45f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04440cdb0}, {0x7fe13dd08ef0}, 0xc0a3d8e700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04440cd20}, 0xc0a3d8e700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}, 0xc025716b80, 0xc0cb3868b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}, 0x3, 0xc0418bf000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b45f980}}}, {0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}, 0xc025716b80, 0xc0418bf000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04440cd20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04440cd20}, {0x40, 0xc00b45fb70}, 0xc00b45fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04440cd20}, {0x4e9d7a0, 0xc0a3d8e700}, 0xc025716b80, 0xc0cb3868b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04440cd20}, 0xc07161fb00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc085179560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc085179560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809835 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc015adaf60}, 0xc032e9a0f0, 0x1, 0xc0133d30a8, 0x519bbc, 0xc07a82b380, 0xc08446b5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc015adaf60}, 0x120, 0xc0133d3230, 0x7a82b320, 0x0, 0x660069, 0xc07a82b380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc015adaf60}, 0xc07a82b320, {{{{0x0, 0x0}, {0xc06f5fc000, 0xeb}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc015adaf60}, 0x6352410, 0x450cbb38) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc015adaf60}, {0xc07a82b200, {0x174301106770b3cb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc015adaf60}, 0xc0536b70e0, 0xc0133d39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc015adaf60}, 0x660089, {0xc07a82b200, {0x174301106770b3cb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc015adaf60}, 0xc064f97700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106770b3cb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106770b3cb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc015adaea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015adaea0}, {0xc0133d53f8, 0x660089}, 0xc0133d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc015adaea0}, {0xc0015221e0}, 0xc064f97500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc015adaa50}, 0xc064f97500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}, 0xc043297360, 0xc034d3d350) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}, 0x3, 0xc045aa5a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0133d5980}}}, {0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}, 0xc043297360, 0xc045aa5a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc015adaa50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015adaa50}, {0x40, 0xc0133d5b70}, 0xc0133d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc015adaa50}, {0x4e9d7a0, 0xc064f97500}, 0xc043297360, 0xc034d3d350) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc015adaa50}, 0xc037e552c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc070adc7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc070adc7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219152288 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219805035 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af298e8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06bfa2480, 0xc057ab3000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06bfa2480, {0xc057ab3000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06bfa2480, {0xc057ab3000, 0x735, 0xc04db61680}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc017bb2818, {0xc057ab3000, 0x7eda79, 0xc057ab3005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0323bbb60, {0xc057ab3000, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0450fc930, {0xc057ab3000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0c5783e78, {0x631b900, 0xc0450fc930}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0c5783c00, {0x7fe42ab59310, 0xc0323bbb60}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0c5783c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0c5783c00, {0xc0a1fda000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc07216d740, {0xc0a1fda000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc056d26b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc056d26b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc056d270e0, {0x634aac0, 0xc056d26b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc056d26a80, 0xc04c7cc890, 0xc07b47f42b, 0xc07b47f42d, {0x635bec0, 0xc03e5d9d40}, 0xc000d6ae00, 0xc040518968, 0xc07b47f429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc056d26a80, {0x63c85b8, 0xc077fee000}, 0xc04c7cc880, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc001d8a044, 0x8}}, 0x0, 0xc067493f80, 0xc067493fb0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc077fee000}, {0x643d810, 0xc0323bbb60}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc050fb8240}, {0x643d810, 0xc0323bbb60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc050fb8240}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 212143004 [sync.Cond.Wait, 12 minutes]: sync.runtime_notifyListWait(0xc028b48050, 0x16) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc03df89500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc03df89b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc066e20000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc066e20000, {0x63c8510, 0xc028b48080}, 0xc0950ebdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc058511aa0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc028b48080}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc058511aa0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808150 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0357c0, {0x63c85b8, 0xc06c284150}, 0xc03178f7e8, 0x1, 0xc005eb30a8, 0x519bbc, 0xc07b21aba0, 0xc0786a1920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06c284150}, 0x120, 0xc005eb3230, 0x7b21ab40, 0x0, 0x660069, 0xc07b21aba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0357c0, {0x63c85b8, 0xc06c284150}, 0xc07b21ab40, {{{{0x0, 0x0}, {0xc0735c4900, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06c284150}, 0x6352410, 0x321a18e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd770, {0x63c85b8, 0xc06c284150}, {0xc07b21aa20, {0x174301105a7a2ce9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd770, {0x63c85b8, 0xc06c284150}, 0xc03544a0a0, 0xc005eb39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06c284150}, 0x660089, {0xc07b21aa20, {0x174301105a7a2ce9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc06c284150}, 0xc01fb8b200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a7a2ce9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a7a2ce9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06c2840c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c2840c0}, {0xc005eb53f8, 0x660089}, 0xc005eb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06c2840c0}, {0xc0015221e0}, 0xc01fb8b000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06c284030}, 0xc01fb8b000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}, 0xc05f0732e0, 0xc027ac84e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}, 0x3, 0xc06c462880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005eb5980}}}, {0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}, 0xc05f0732e0, 0xc06c462880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06c284030}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c284030}, {0x40, 0xc005eb5b70}, 0xc005eb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06c284030}, {0x4e9d7a0, 0xc01fb8b000}, 0xc05f0732e0, 0xc027ac84e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06c284030}, 0xc04b706a20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07b21a900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07b21a900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801439 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc091064548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0910645b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0bd743bf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc052246e00, {0xc04cbe1880, 0x2b, 0x6f}, {0xc012f74d80, 0x2b, 0x7f}, 0x1, 0x3b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0bd743a88, {0xc04cbe1880, 0x1, 0x7f}, {0xc012f74d80, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0bd743a88, {0xc04cbe1880, 0x2b, 0x6f}, {0xc012f74d80, 0x2b, 0x7f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0bd7431c0, {0xc012f74d80, 0x7f, 0x6f}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0bd7431c0, {0xc04cbe1880, 0x2b, 0xc012f74d80}, {0xc012f74d80, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0bd742f00, {0xc012f74d80, 0x2b, 0x7f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06daa6688, {{0xc004d8ee10, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04e8ca100, {{0xc004d8ee10, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07211bc00, {0x63c85b8, 0xc0679c0ff0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0679c0ff0}, {0x64a6560, 0xc04e8ca100}, {0xc004d8ee10, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0679c0ff0}, {0x6466ed8, 0xc06daa6680}, {0xc004d8ee10, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0397665a0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0845e1400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c95400, {0x63c85b8, 0xc0679c0ff0}, 0xc0845e1400, 0xc0468850e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c95400, {0x63c85b8, 0xc0679c0ff0}, 0xc0845e1400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0679c0f00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0679c0f00}, {0xc0935988f0, 0x660089}, 0xc0935988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0679c0f00}, {0xc00102e4e0}, 0xc0845e1300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0679c0ea0}, 0xc0845e1300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02ef02990}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800990 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00e450280, {0x63c85b8, 0xc0750170b0}, 0xc04252cbe8, 0x1, 0xc04fcf50a8, 0x519bbc, 0xc02201a840, 0xc0682244e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0750170b0}, 0x120, 0xc04fcf5230, 0x2201a7e0, 0x0, 0x660069, 0xc02201a840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00e450280, {0x63c85b8, 0xc0750170b0}, 0xc02201a7e0, {{{{0x0, 0x0}, {0xc073d94900, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0750170b0}, 0x6352410, 0x11f10) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc000b61d10, {0x63c85b8, 0xc0750170b0}, {0xc02201a6c0, {0x174301101aa4b463, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc000b61d10, {0x63c85b8, 0xc0750170b0}, 0xc049d0cc80, 0xc04fcf59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0750170b0}, 0x660089, {0xc02201a6c0, {0x174301101aa4b463, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00077dc00, {0x63c85b8, 0xc0750170b0}, 0xc04b2a1400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101aa4b463, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101aa4b463, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075017020}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075017020}, {0xc04fcf73f8, 0x660089}, 0xc04fcf7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075017020}, {0xc087f10eb8}, 0xc04b2a1200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc075016f90}, 0xc04b2a1200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}, 0xc0210d51a0, 0xc04e078978) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}, 0x3, 0xc047aa47c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04fcf7980}}}, {0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}, 0xc0210d51a0, 0xc047aa47c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc075016f90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075016f90}, {0x40, 0xc04fcf7b70}, 0xc04fcf7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc075016f90}, {0x4e9d7a0, 0xc04b2a1200}, 0xc0210d51a0, 0xc04e078978) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc075016f90}, 0xc026043f20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc022d64480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc022d64480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807305 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6000, {0x63c85b8, 0xc043061380}, 0xc065e05aa0, 0x1, 0xc0042ab0a8, 0x519bbc, 0xc044e74240, 0xc06b9a01e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0546ac7e0, {0x63c85b8, 0xc043061380}, 0xc0546ac810, 0xc0042ab1a8, 0x41, 0x60, 0xffffffffffffffff, 0xc044e74240, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6000, {0x63c85b8, 0xc043061380}, 0xc044e74000, {{{{0xc040439680, 0x2}, {0xc03b87d200, 0x2}}, {{0xc04166a900, 0x8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc043061380}, 0x63521a0, 0x12e3c768) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddb30, {0x63c85b8, 0xc043061380}, {0xc07b8f85a0, {0x1743010ffc5f47fd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc043061380}, 0xc04c1c4960, 0xc0042ab9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc043061380}, 0x660089, {0xc07b8f85a0, {0x1743010ffc5f47fd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc043061380}, 0xc08d7f3800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffc5f47fd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffc5f47fd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0430612f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0430612f0}, {0xc0042ad3f8, 0x660089}, 0xc0042ad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0430612f0}, {0xfef486faa9080a2a}, 0xc08d7f3600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0430611a0}, 0xc08d7f3600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}, 0xc062506080, 0xc0a55fe4c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}, 0x3, 0xc00ea59480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0042ad980}}}, {0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}, 0xc062506080, 0xc00ea59480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0430611a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0430611a0}, {0x40, 0xc0042adb70}, 0xc0042adab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0430611a0}, {0x4e9d7a0, 0xc08d7f3600}, 0xc062506080, 0xc0a55fe4c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0430611a0}, 0xc0474f80c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07b8f8000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07b8f8000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795506 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc08f15e548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc08f15e5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc090eaecf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc081b92380, {0xc00d7b2a80, 0x2b, 0x6d}, {0xc018a54150, 0x2b, 0x6d}, 0x1, 0xec) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc090eaeb88, {0xc00d7b2a80, 0x1, 0x6d}, {0xc018a54150, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc090eaeb88, {0xc00d7b2a80, 0x2b, 0x6d}, {0xc018a54150, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc090eae2c0, {0xc018a54150, 0x6d, 0x6d}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc090eae2c0, {0xc00d7b2a80, 0x2b, 0xc018a54150}, {0xc018a54150, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc090eae000, {0xc018a54150, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0532e5088, {{0xc06cc21290, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07a4c3e00, {{0xc06cc21290, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08c161400, {0x63c85b8, 0xc0c35c08a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0c35c08a0}, {0x64a6560, 0xc07a4c3e00}, {0xc06cc21290, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0c35c08a0}, {0x6466ed8, 0xc0532e5080}, {0xc06cc21290, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01cfdd848}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0ce07f400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020091400, {0x63c85b8, 0xc0c35c08a0}, 0xc0ce07f400, 0xc070de1360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020091400, {0x63c85b8, 0xc0c35c08a0}, 0xc0ce07f400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c35c0810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c35c0810}, {0xc0ad83c8f0, 0x660089}, 0xc0ad83c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c35c0810}, {0xc00102e4e0}, 0xc0ce07f300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c35c07b0}, 0xc0ce07f300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0710d5350}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812804 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219796888 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0880c0548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0880c05b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc00e8b56f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc05a6aa380, {0xc025c6f900, 0x2b, 0x74}, {0xc01c311e30, 0x2b, 0x6e}, 0x1, 0x56) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc00e8b5588, {0xc025c6f900, 0x1, 0x6e}, {0xc01c311e30, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc00e8b5588, {0xc025c6f900, 0x2b, 0x74}, {0xc01c311e30, 0x2b, 0x6e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc00e8b4cc0, {0xc01c311e30, 0x6e, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc00e8b4cc0, {0xc025c6f900, 0x2b, 0xc01c311e30}, {0xc01c311e30, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc00e8b4a00, {0xc01c311e30, 0x2b, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04a525608, {{0xc04cd1b6e0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc074cf4200, {{0xc04cd1b6e0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0482bf400, {0x63c85b8, 0xc02e97f2c0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02e97f2c0}, {0x64a6560, 0xc074cf4200}, {0xc04cd1b6e0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02e97f2c0}, {0x6466ed8, 0xc04a525600}, {0xc04cd1b6e0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc042f81158}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04208e700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020465400, {0x63c85b8, 0xc02e97f2c0}, 0xc04208e700, 0xc039b4a960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020465400, {0x63c85b8, 0xc02e97f2c0}, 0xc04208e700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02e97f050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02e97f050}, {0xc002fea8f0, 0x660089}, 0xc002fea8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02e97f050}, {0xc00102e4e0}, 0xc04208e600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02e97eff0}, 0xc04208e600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc041498630}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810821 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc04cd0a090}, 0xc0d5736798, 0x1, 0xc00dfcd6b8, 0x519bbc, 0xc0504322a0, 0xc0674893e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc02b8287b0, {0x63c85b8, 0xc04cd0a090}, 0xc060fc6d80, 0xc00dfcd7b8, 0x50432300, 0x0, 0x0, 0xc0504322a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc04cd0a090}, 0xc050432240, {{{{0xc044876480, 0x10}, {0xc03dcee000, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04cd0a090}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc04cd0a090}, {0x0, {0x174301106f5991f5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc04cd0a090}, 0xc06269e640, 0xc00dfce000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04cd0a090}, 0x660089, {0x0, {0x174301106f5991f5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc04cd0a090}, 0xc0d233ee00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f5991f5, 0x0, 0x0}, 0xc04c35c6b0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f5991f5, 0x0, 0x0}, 0xc04c35c6b0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04cd0a000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cd0a000}, {0xc07c2e5a08, 0x660089}, 0xc07c2e5a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04cd0a000}, {0xc00102e4e0}, 0xc0d233ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b8b5f80}, 0xc0d233ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc06b8b5ef0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc06b8b5ef0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807356 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0171c1020}, 0xc04da06cb0, {0x646cc48, 0xc0171c1080}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0171c1020}, 0xedb7a8350, 0xc054a40120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0171c1020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07c076b70}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc04da06ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc07c076b70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812247 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc04bb9a180}, 0xc029c0a3d8, 0x1, 0xc03973b0a8, 0x519bbc, 0xc01088b800, 0xc05153d140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04bb9a180}, 0x120, 0xc03973b230, 0x1088b7a0, 0x0, 0x660069, 0xc01088b800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc04bb9a180}, 0xc01088b7a0, {{{{0x0, 0x0}, {0xc01e23b8c0, 0xe}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04bb9a180}, 0x6352410, 0x54d3a188) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc04bb9a180}, {0xc01088b560, {0x174301107f0ff4ae, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc04bb9a180}, 0xc065d64460, 0xc03973b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04bb9a180}, 0x660089, {0xc01088b560, {0x174301107f0ff4ae, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc04bb9a180}, 0xc08a8d2700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f0ff4ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f0ff4ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bb9a0f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bb9a0f0}, {0xc03973d3f8, 0x660089}, 0xc03973d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bb9a0f0}, {0x466d4d336b6a5a35}, 0xc08a8d2500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bb9a060}, 0xc08a8d2500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}, 0xc06d764e60, 0xc044b4a300) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}, 0x3, 0xc06e1061c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03973d980}}}, {0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}, 0xc06d764e60, 0xc06e1061c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bb9a060}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bb9a060}, {0x40, 0xc03973db70}, 0xc03973dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bb9a060}, {0x4e9d7a0, 0xc08a8d2500}, 0xc06d764e60, 0xc044b4a300) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bb9a060}, 0xc032c0b8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03fdb07e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03fdb07e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809792 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021c68c00, {0x63c85b8, 0xc090d396b0}, 0xc05adca400, 0xc07cfa7ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c68c00, {0x63c85b8, 0xc090d396b0}, 0xc05adca400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067536073, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067536073, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc090d39620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090d39620}, {0xc00e14d3f8, 0x660089}, 0xc00e14d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc090d39620}, {0x7fe3b5951e28}, 0xc05adca200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc090d39590}, 0xc05adca200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}, 0xc03248ac80, 0xc02132b248) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}, 0x3, 0xc05b24bf80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e14d980}}}, {0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}, 0xc03248ac80, 0xc05b24bf80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc090d39590}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090d39590}, {0x40, 0xc00e14db70}, 0xc00e14dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc090d39590}, {0x4e9d7a0, 0xc05adca200}, 0xc03248ac80, 0xc02132b248) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc090d39590}, 0xc02e17c2a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc046ed6fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc046ed6fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800615 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x55) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc054fe8cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0515d8000, {0xc01666d300, 0x32, 0x7e}, {0xc082f5d620, 0x32, 0x5d}, 0x1, 0x8c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc054fe8b88, {0xc01666d300, 0x1, 0x5d}, {0xc082f5d620, 0x2cf, 0xc0954f4790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc054fe8b88, {0xc01666d300, 0x32, 0x7e}, {0xc082f5d620, 0x32, 0x5d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc054fe82c0, {0xc082f5d620, 0x5d, 0x7e}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc054fe82c0, {0xc01666d300, 0x5d, 0xc082f5d620}, {0xc082f5d620, 0xc060c1b710, 0x31}, 0x82) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc054fe8000, {0xc082f5d620, 0x32, 0x5d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06606e2a8, {{0xc060c1b710, 0x31, 0x82}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05779b900, {{0xc0b98fb3b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc097f34400, {0x63c85b8, 0xc04dbc3830}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04dbc3830}, {0x64a6560, 0xc05779b900}, {0xc0b98fb3b0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04dbc3830}, {0x6466ed8, 0xc06606e000}, {0xc0b98fb3b0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c77f7a0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06b0a9700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b10400, {0x63c85b8, 0xc04dbc3830}, 0xc06b0a9700, 0xc07a98a6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b10400, {0x63c85b8, 0xc04dbc3830}, 0xc06b0a9700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04dbc37a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04dbc37a0}, {0xc07d2528f0, 0x660089}, 0xc07d2528f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04dbc37a0}, {0xc00102e4e0}, 0xc06b0a9600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04dbc3740}, 0xc06b0a9600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06b4b8e10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805413 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc00005e5a0}, 0xc0314e47e0, 0x1, 0xc0871d10a8, 0x519bbc, 0xc05a519500, 0xc05827eba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc00005e5a0}, 0x1e0, 0xc0871d1230, 0x5a5194a0, 0xc05b7fe820, 0x18, 0xc05a519500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc00005e5a0}, 0xc05a5194a0, {{{{0x0, 0x0}, {0xc057fd3200, 0x43}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x1cb875892805cba8, {0x63c85b8, 0xc00005e5a0}, 0xa00b0000100b00, 0x202800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc00005e5a0}, {0xc05bd9bc20, {0x174301103e3dd4d7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc00005e5a0}, 0xc034a48640, 0xc0871d19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc00005e5a0}, 0x660089, {0xc05bd9bc20, {0x174301103e3dd4d7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc00005e5a0}, 0xc0ab175d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103e3dd4d7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103e3dd4d7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc00005e510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00005e510}, {0xc0871d33f8, 0x660089}, 0xc0871d3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc00005e510}, {0xc0015221e0}, 0xc0ab175b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc00005e480}, 0xc0ab175b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}, 0xc014a7ec00, 0xc0314e5620) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}, 0x3, 0xc027513080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0871d3980}}}, {0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}, 0xc014a7ec00, 0xc027513080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc00005e480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00005e480}, {0x40, 0xc0871d3b70}, 0xc0871d3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}, 0x203009, 0x203009) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc00005e480}, {0x4e9d7a0, 0xc0ab175b00}, 0xc014a7ec00, 0xc0314e5620) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc00005e480}, 0xc030cd3ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05bd9bb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05bd9bb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807685 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc023145fb0}, 0xc09c7faa00, 0xc020fc7e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc023145fb0}, 0xc09c7faa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058bf3457, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058bf3457, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023145f20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023145f20}, {0xc007a0f3f8, 0x660089}, 0xc007a0f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023145f20}, {0x1}, 0xc09c7fa800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc023145e30}, 0xc09c7fa800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}, 0xc022fdb420, 0xc03fe1ca38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}, 0x3, 0xc059c36b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007a0f980}}}, {0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}, 0xc022fdb420, 0xc059c36b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc023145e30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023145e30}, {0x40, 0xc007a0fb70}, 0xc007a0fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc023145e30}, {0x4e9d7a0, 0xc09c7fa800}, 0xc022fdb420, 0xc03fe1ca38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc023145e30}, 0xc0d500f5c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc058adb440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc058adb440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219787962 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc06387fa20, {0x63c85b8, 0xc02a0b7380}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc044b16600, {0x63c85b8, 0xc02a0b7380}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc044b16600, {0x63c8510, 0xc05c332940}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc044b16600, {0x63c8510, 0xc05c332940}, {0x63f7db8, 0x9bf6f48}, 0xc0914bd088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc044b16600, {0x63c85b8, 0xc031f604b0}, {{0x63f7db8, 0x9bf6f48}, {0xc0723c9283, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc044b16600, {0x63c85b8, 0xc031f604b0}, {{0x63f7db8, 0x9bf6f48}, {0xc0723c9283, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0723c9283, 0x6}, 0x0, 0x0}, {0x2423e6cf, 0xedb7a834f, 0x0}, {0x2423f017, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc044b16600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc044b16600, {0x63c8510, 0xc0838abac0}, 0xc053b77dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04553d100) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0838abac0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04553d100) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804326 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc05a6bcc00}, 0xc035097e18, 0x1, 0xc079feb0a8, 0x519bbc, 0xc04f048240, 0xc04e0769c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc05a6bcc00}, 0x1e0, 0xc079feb230, 0x4f0481e0, 0x0, 0x660069, 0xc04f048240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc05a6bcc00}, 0xc04f0481e0, {{{{0x0, 0x0}, {0xc0756ce6c0, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc05a6bcc00}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc05a6bcc00}, {0xc056fdf440, {0x17430110329b0990, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc05a6bcc00}, 0xc03a635e00, 0xc079feb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05a6bcc00}, 0x660089, {0xc056fdf440, {0x17430110329b0990, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc05a6bcc00}, 0xc0579e8100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110329b0990, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110329b0990, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05a6bcb70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05a6bcb70}, {0xc079fed3f8, 0x660089}, 0xc079fed400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05a6bcb70}, {0x7fe3bdd54318}, 0xc00d8eff00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05a6bcae0}, 0xc00d8eff00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}, 0xc01a53e6e0, 0xc0117df560) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}, 0x3, 0xc03fbe7a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079fed980}}}, {0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}, 0xc01a53e6e0, 0xc03fbe7a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05a6bcae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05a6bcae0}, {0x40, 0xc079fedb70}, 0xc079fedab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05a6bcae0}, {0x4e9d7a0, 0xc00d8eff00}, 0xc01a53e6e0, 0xc0117df560) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05a6bcae0}, 0xc01e36e300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc056fdf320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc056fdf320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812502 [semacquire]: sync.runtime_SemacquireMutex(0xc024606000, 0x3, 0xc024606000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074b2c0) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc083122b68) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074b2b0, 0x11869e5, 0x7fe219543740, 0x7fa3, 0x7fe1d7cc4780) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc0ba4d39c0, 0x7fe219543701, 0x7fe219543760, 0x3ab3, 0x3ab8) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc03e653600, {0xb500a, 0x3ab3}, 0x0, 0xc06158e978) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc06158e700, {0xffffffffffffff15, 0xc083122df0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc06158e700) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc06158e700, {0xc068a3f0e0, 0x1f, 0x1f}, 0xc02ed2ee80, 0x50) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc06158e700, {0xc02ed2ee80, 0x1f, 0x77}, {0xc068a3f0e0, 0x1f, 0x1f}, 0x0, 0xc3) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07432c1e0, {0xc02ed2ee80, 0x1, 0x1f}, {0xc068a3f0e0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07432c1e0, {0xc02ed2ee80, 0x1f, 0x77}, {0xc068a3f0e0, 0x1f, 0x1f}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07432b6c0, {0xc068a3f0e0, 0x4a2372, 0x1f}, 0x464ed60, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07432b6c0, {0xc02ed2ee80, 0x1f, 0x1f}, {0xc068a3f0e0, 0x1e, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07432b400, {0xc068a3f0e0, 0x1f, 0x1f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0862f4c28, {{0xc068a3f0c0, 0x1e, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07cafd000, {0x63c85b8, 0xc04ae6b0b0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04ae6b0b0}, {0x64a6618, 0xc0862f4c28}, {0xc068a3f0c0, 0x0, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04ae6b0b0}, {0x7fe429f83df0, 0xc034107480}, {0xc068a3f0c0, 0x1e, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x63c85b8, 0xc04ae6b0b0}, {0x7fe429f83df0, 0xc034107480}, {0xc068a3f0c0, 0xc083123d10, 0x461247}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:617 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0, {0x63c85b8, 0xc04ae6b0b0}, {0x7fe429f83df0, 0xc034107480}, {0x2c, 0xeb, 0x63, 0x36, 0xca, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:98 +0xdb github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0x2c, 0xeb, 0x63, 0x36, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:1958 +0xbc github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05707ea00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:201 +0x37e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:644 +0x190 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:612 +0x2da github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:428 +0x5f3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc0244e8800, {0x63c85b8, 0xc04ae6b0b0}, {0xc06066c2c8, 0x8}, 0xc05707ea00, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:673 +0x1c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x0, {0x63c85b8, _}, {_, _}, _, {{{0x1742daf133a7d2e7, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:773 +0xaa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc0244e8800, {0x63c85b8, _}, _, _, {{{0x1742daf133a7d2e7, 0x0, 0x0}, 0x0, {0x1f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:113 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0244e8800, {0x63c85b8, 0xc04ae6b0b0}, 0xc05707ea00, 0xc064890a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:167 +0x7b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0244e8800, {0x63c85b8, 0xc04ae6b0b0}, 0xc05707ea00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ae6aff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ae6aff0}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ae6aff0}, {0xc00102e4e0}, 0xc05707e900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ae6af90}, 0xc05707e900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc018086db0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219694729 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe795e8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05b23d580, 0xc08afa8000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05b23d580, {0xc08afa8000, 0x2b88, 0x2b88}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05b23d580, {0xc08afa8000, 0x2b88, 0xc064142500}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0679d0608, {0xc08afa8000, 0x7eda79, 0xc08afa8005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc047da9740, {0xc08afa8000, 0x3, 0x2b88}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc00b73e618, {0xc08afa8000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0d0545078, {0x631b900, 0xc00b73e618}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0d0544e00, {0x7fe42ab59310, 0xc047da9740}, 0x2b88) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0d0544e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0d0544e00, {0xc05158c000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0a6a82300, {0xc05158c000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc08e6ac088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc08e6ac088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc08e6ac660, {0x634aac0, 0xc08e6ac088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc08e6ac000, 0xc04a545ef0, 0xc07181542b, 0xc07181542d, {0x635bec0, 0xc00edaa750}, 0xc000d6ae00, 0xc02809e3d8, 0xc071815429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc08e6ac000, {0x63c85b8, 0xc074598900}, 0xc04a545ee0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08b575009, 0x8}}, 0x0, 0xc0745988a0, 0xc0745988d0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc074598900}, {0x643d810, 0xc047da9740}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0cd5421b0}, {0x643d810, 0xc047da9740}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0cd5421b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806125 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc062b75440}, 0xc0bdd26ed0, 0x1, 0xc007e290a8, 0x519bbc, 0xc05a1a7aa0, 0xc01a184f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc062b75440}, 0x1e0, 0xc007e29230, 0x5a1a7a40, 0x0, 0x660069, 0xc05a1a7aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc062b75440}, 0xc05a1a7a40, {{{{0x0, 0x0}, {0xc04bea8fc0, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc022e3cde0, {0x63c85b8, 0xc062b75440}, 0xc01e6e4350, 0x7e29428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc062b75440}, {0xc04e687c20, {0x174301104c6e8840, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc062b75440}, 0xc06f8de140, 0xc007e299f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc062b75440}, 0x660089, {0xc04e687c20, {0x174301104c6e8840, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc062b75440}, 0xc08bc82c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c6e8840, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c6e8840, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062b753b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062b753b0}, {0xc007e2b3f8, 0x660089}, 0xc007e2b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062b753b0}, {0x3241382f6e6f2b75}, 0xc08bc82a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062b75320}, 0xc08bc82a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}, 0xc073ecdc80, 0xc089f8d4b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}, 0x3, 0xc071761e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007e2b980}}}, {0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}, 0xc073ecdc80, 0xc071761e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062b75320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062b75320}, {0x40, 0xc007e2bb70}, 0xc007e2bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062b75320}, {0x4e9d7a0, 0xc08bc82a00}, 0xc073ecdc80, 0xc089f8d4b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062b75320}, 0xc04bc0f800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04e687b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04e687b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800382 [runnable]: sync.runtime_SemacquireMutex(0xc0337de690, 0x8f, 0x7fd692a51ac0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0337de538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0337de560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0455b5680, {0xffffffffffffff15, 0xc0337de6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0455b5680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0455b5680, {0xc0894bbf00, 0x32, 0x79}, {0xc0251c96c0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc094760898, {0xc0894bbf00, 0x32, 0x79}, {0xc0251c96c0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09475f1c0, {0xc0251c96c0, 0x32, 0x79}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09475f1c0, {0xc0894bbf00, 0x32, 0xc0251c96c0}, {0xc0251c96c0, 0xc02f2790e0, 0x31}, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09475ef00, {0xc0251c96c0, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc09c449e28, {{0xc02f2790e0, 0x31, 0x60}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0137b3f00, {{0xc0476e72f0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc051ae8800, {0x63c85b8, 0xc062482000}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062482000}, {0x64a6560, 0xc0137b3f00}, {0xc0476e72f0, 0xc03b3ac6f0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062482000}, {0x6466ed8, 0xc09c449b80}, {0xc0476e72f0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02e7ce7e0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0537d4200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020abc800, {0x63c85b8, 0xc062482000}, 0xc0537d4200, 0xc045fde500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020abc800, {0x63c85b8, 0xc062482000}, 0xc0537d4200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc054761f50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054761f50}, {0xc0025128f0, 0x660089}, 0xc0025128f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc054761f50}, {0x24}, 0xc0537d4000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc054761ef0}, 0xc0537d4000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b830}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802078 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc03d3c7c20}, 0xc002ef8cb0, {0x646cc48, 0xc03d3c7c80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc03d3c7c20}, 0xedb7a834f, 0xc0196cd8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc03d3c7c20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03f22e0c0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc002ef8ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc03f22e0c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219150685 [select, 1 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 219810343 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0305badc0, {0x63c85b8, 0xc045468030}, 0xc03c04eca8, 0x1, 0xc026f010a8, 0x519bbc, 0xc0161ddb60, 0xc0616e3140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc045468030}, 0x120, 0xc026f01230, 0x161ddb00, 0x0, 0x660069, 0xc0161ddb60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0305badc0, {0x63c85b8, 0xc045468030}, 0xc0161ddb00, {{{{0x0, 0x0}, {0xc013344480, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc045468030}, 0x6352410, 0x412aca80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030586e60, {0x63c85b8, 0xc045468030}, {0xc0161dd9e0, {0x174301106c367d6e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030586e60, {0x63c85b8, 0xc045468030}, 0xc02bc90500, 0xc026f019f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc045468030}, 0x660089, {0xc0161dd9e0, {0x174301106c367d6e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0305d3000, {0x63c85b8, 0xc045468030}, 0xc08c78c300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c367d6e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c367d6e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027a03f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027a03f80}, {0xc026f033f8, 0x660089}, 0xc026f03400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027a03f80}, {0x7fe3b7b23a40}, 0xc08c78c100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027a03ef0}, 0xc08c78c100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}, 0xc08bdf2080, 0xc04a2d55f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}, 0x3, 0xc0bdce3240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026f03980}}}, {0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}, 0xc08bdf2080, 0xc0bdce3240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc027a03ef0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027a03ef0}, {0x40, 0xc026f03b70}, 0xc026f03ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc027a03ef0}, {0x4e9d7a0, 0xc08c78c100}, 0xc08bdf2080, 0xc04a2d55f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc027a03ef0}, 0xc02de2cf00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc023399b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc023399b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808129 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc03a963530}, 0xc06da48a00, 0xc023e39860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc03a963530}, 0xc06da48a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c49876b, 0x0, 0x0}, 0xc04fa00360, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c49876b, 0x0, 0x0}, 0xc04fa00360, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03a9634a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a9634a0}, {0xc00a4473f8, 0x660089}, 0xc00a447400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03a9634a0}, {0xc0015221e0}, 0xc06da48800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a963410}, 0xc06da48800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}, 0xc0d475b2c0, 0xc03081bba8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}, 0x3, 0xc029491800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a447980}}}, {0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}, 0xc0d475b2c0, 0xc029491800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03a963410}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a963410}, {0x40, 0xc00a447b70}, 0xc00a447ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03a963410}, {0x4e9d7a0, 0xc06da48800}, 0xc0d475b2c0, 0xc03081bba8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03a963410}, 0xc06f809e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059e98d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059e98d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803667 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc0745918c0}, 0xc0643619e0, 0x1, 0x0, 0x1, 0xc03c285da0, 0xc082cd0840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0745918c0}, 0x120, 0xc0766d2728, 0x3c285d40, 0x0, 0x660069, 0xc03c285da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc0745918c0}, 0xc03c285d40, {{{{0x0, 0x0}, {0xc0731325a0, 0x17}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0745918c0}, 0x6352410, 0x34330568) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc0745918c0}, {0xc03b940c60, {0x174301100d71b895, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc0745918c0}, 0xc078118640, 0xc076856ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0745918c0}, 0x660089, {0xc03b940c60, {0x174301100d71b895, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc0745918c0}, 0xc0d503eb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100d71b895, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100d71b895, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074591830}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074591830}, {0xc01fad48f0, 0x660089}, 0xc01fad48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074591830}, {0xc00102e4e0}, 0xc0d503ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0745917d0}, 0xc0d503ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc037feb410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802474 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc047498c00}, 0xc0b4407e00, 0xc06c785cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc047498c00}, 0xc0b4407e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff7940e2e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff7940e2e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047498b70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047498b70}, {0xc01f2f93f8, 0x660089}, 0xc01f2f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047498b70}, {0xc0015221e0}, 0xc0b4407c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047498ab0}, 0xc0b4407c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}, 0xc08c928860, 0xc01ff39410) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}, 0x3, 0xc0d43ee880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f2f9980}}}, {0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}, 0xc08c928860, 0xc0d43ee880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047498ab0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047498ab0}, {0x40, 0xc01f2f9b70}, 0xc01f2f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047498ab0}, {0x4e9d7a0, 0xc0b4407c00}, 0xc08c928860, 0xc01ff39410) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047498ab0}, 0xc047214c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04b5c37a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04b5c37a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802157 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc04d17f4a0}, 0xc0479376e0, 0x1, 0xc01d5390a8, 0x519bbc, 0xc014a74720, 0xc0385d1020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04d17f4a0}, 0x120, 0xc01d539230, 0x14a746c0, 0x0, 0x660069, 0xc014a74720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc04d17f4a0}, 0xc014a746c0, {{{{0x0, 0x0}, {0xc03dcef8c0, 0x39}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d17f4a0}, 0x6352410, 0x351a99c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc04d17f4a0}, {0xc014a74480, {0x174301101b171816, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc04d17f4a0}, 0xc046884d20, 0xc01d5399f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d17f4a0}, 0x660089, {0xc014a74480, {0x174301101b171816, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc04d17f4a0}, 0xc07e270000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b171816, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b171816, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d17f410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d17f410}, {0xc01d53b3f8, 0x660089}, 0xc01d53b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d17f410}, {0x36}, 0xc06d00fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d17f320}, 0xc06d00fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}, 0xc04ecdb380, 0xc0a74182d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}, 0x3, 0xc06e1c7ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01d53b980}}}, {0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}, 0xc04ecdb380, 0xc06e1c7ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d17f320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d17f320}, {0x40, 0xc01d53bb70}, 0xc01d53bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d17f320}, {0x4e9d7a0, 0xc06d00fe00}, 0xc04ecdb380, 0xc0a74182d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d17f320}, 0xc01ad184e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc020e38360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc020e38360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219773795 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff35df0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc08d74e280, 0xc00d5cc000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc08d74e280, {0xc00d5cc000, 0x5639, 0x5639}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc08d74e280, {0xc00d5cc000, 0x0, 0x0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc032cb25c8, {0xc00d5cc000, 0x2faf080, 0x3b9aca00}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04f8e7da0, {0xc00d5cc000, 0x0, 0x5639}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04ebf62d0, {0xc00d5cc000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc034d9c278, {0x631b900, 0xc04ebf62d0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc034d9c000, {0x7fe42ab59310, 0xc04f8e7da0}, 0x0) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc034d9c000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc034d9c000, {0xc07cd4a000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc028959050, {0xc07cd4a000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc014d2e088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc014d2e088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc014d2e660, {0x634aac0, 0xc014d2e088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc014d2e000, 0xc07b99fd60, 0xc077f3342b, 0xc077f3342d, {0x635bec0, 0xc04d329200}, 0xc000d6ae00, 0xc0a03a0c38, 0xc077f33429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc014d2e000, {0x63c85b8, 0xc03db45650}, 0xc07b99fd50, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a5f39037, 0x8}}, 0x0, 0xc03db455f0, 0xc03db45620, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03db45650}, {0x643d810, 0xc04f8e7da0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02f527ad0}, {0x643d810, 0xc04f8e7da0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02f527ad0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219775455 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc062094f10, 0x64) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc06d913500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc06d913b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc011d33900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc011d33900, {0x63c8510, 0xc062094f40}, 0xc08af7fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05a8bd130) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc062094f40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05a8bd130) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802587 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04c980150}, 0xc04621b620, 0x1, 0xc02120b0a8, 0x519bbc, 0xc038294a80, 0xc055f375c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04c980150}, 0x120, 0xc02120b230, 0x38294a20, 0x0, 0x660069, 0xc038294a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04c980150}, 0xc038294a20, {{{{0x0, 0x0}, {0xc08d00c900, 0xcb}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04c980150}, 0x6352410, 0x6492b320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04c980150}, {0xc038294900, {0x174301101c76aae4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04c980150}, 0xc0bcee0640, 0xc02120b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c980150}, 0x660089, {0xc038294900, {0x174301101c76aae4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04c980150}, 0xc04d4c9f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c76aae4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c76aae4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c9800c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c9800c0}, {0xc02120d3f8, 0x660089}, 0xc02120d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c9800c0}, {0x0}, 0xc04d4c9c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc00e3b7ec0}, 0xc04d4c9c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}, 0xc0533a3440, 0xc0571d4948) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}, 0x3, 0xc07c409900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02120d980}}}, {0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}, 0xc0533a3440, 0xc07c409900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc00e3b7ec0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00e3b7ec0}, {0x40, 0xc02120db70}, 0xc02120dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc00e3b7ec0}, {0x4e9d7a0, 0xc04d4c9c00}, 0xc0533a3440, 0xc0571d4948) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc00e3b7ec0}, 0xc01e3f4060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0382947e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0382947e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800869 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01853c800, {0x63c85b8, 0xc06dbb8090}, 0xc0401eb100, 0xc068236320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01853c800, {0x63c85b8, 0xc06dbb8090}, 0xc0401eb100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011016cf00c4, 0x0, 0x0}, 0xc063dcf320, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011016cf00c4, 0x0, 0x0}, 0xc063dcf320, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04b741f50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04b741f50}, {0xc0766d3918, 0x660089}, 0xc0766d3920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04b741f50}, {0xc00102e4e0}, 0xc0401eb000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04b741ef0}, 0xc0401eb000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc019cc6b40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04b741dd0}, {0xc00dfd36b0, 0x24}, 0x4828cd1a2, 0xc00776ed40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc079be7470}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04b741dd0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812751 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802263 [runnable]: sync.runtime_SemacquireMutex(0xc09a79a690, 0x8f, 0xc09a79a6a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc09a79a538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc09a79a560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc084757680, {0xffffffffffffff15, 0xc09a79a6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc084757680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc084757680, {0xc02fd5c840, 0x32, 0x32}, {0xc02fd5c800, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092d7b998, {0xc02fd5c840, 0x32, 0x32}, {0xc02fd5c800, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092d7a2c0, {0xc02fd5c800, 0x32, 0x32}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092d7a2c0, {0xc02fd5c840, 0x32, 0xc02fd5c800}, {0xc02fd5c800, 0xc0480777c0, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092d7a000, {0xc02fd5c800, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0d3eaeea8, {{0xc0480777c0, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0a3a99e00, {{0xc019cd8900, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc025f3c800, {0x63c85b8, 0xc04e4bd380}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04e4bd380}, {0x64a6560, 0xc0a3a99e00}, {0xc019cd8900, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04e4bd380}, {0x6466ed8, 0xc0d3eaec00}, {0xc019cd8900, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0a6a83128}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0cd6e7a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ede400, {0x63c85b8, 0xc04e4bd380}, 0xc0cd6e7a00, 0xc04e2d1860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ede400, {0x63c85b8, 0xc04e4bd380}, 0xc0cd6e7a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e4bd2f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e4bd2f0}, {0xc0562c48f0, 0x660089}, 0xc0562c48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e4bd2f0}, {0xc00102e4e0}, 0xc0cd6e7900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e4bd290}, 0xc0cd6e7900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0bb639590}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812002 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc06199c4e0}, 0xc01dae5d40, 0x1, 0xc0182850a8, 0x519bbc, 0xc065f5c600, 0xc04565d800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06199c4e0}, 0x120, 0xc018285230, 0x65f5c5a0, 0x0, 0x660069, 0xc065f5c600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc06199c4e0}, 0xc065f5c5a0, {{{{0x0, 0x0}, {0xc0b70a2b40, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06199c4e0}, 0x6352410, 0x33a923a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc06199c4e0}, {0xc065f5c480, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc06199c4e0}, 0xc07076b900, 0xc0182859f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06199c4e0}, 0x660089, {0xc065f5c480, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc06199c4e0}, 0xc060cc9700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06199c450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06199c450}, {0xc0182873f8, 0x660089}, 0xc018287400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06199c450}, {0x7265765f77736d72}, 0xc060cc9500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06199c3c0}, 0xc060cc9500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}, 0xc0262c6540, 0xc013d6c9c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}, 0x3, 0xc04e09e500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc018287980}}}, {0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}, 0xc0262c6540, 0xc04e09e500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06199c3c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06199c3c0}, {0x40, 0xc018287b70}, 0xc018287ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06199c3c0}, {0x4e9d7a0, 0xc060cc9500}, 0xc0262c6540, 0xc013d6c9c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06199c3c0}, 0xc063da7260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc065f5c360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc065f5c360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799755 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc039481050) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0394810b8, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0888740f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0be731500, {0xc02c28f280, 0x2b, 0x79}, {0xc04cc1f8f0, 0x2b, 0x6f}, 0x1, 0x40) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc088873f88, {0xc02c28f280, 0x1, 0x6f}, {0xc04cc1f8f0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc088873f88, {0xc02c28f280, 0x2b, 0x79}, {0xc04cc1f8f0, 0x2b, 0x6f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0888736c0, {0xc04cc1f8f0, 0x6f, 0x79}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0888736c0, {0xc02c28f280, 0x2b, 0xc04cc1f8f0}, {0xc04cc1f8f0, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc088873400, {0xc04cc1f8f0, 0x2b, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc072e28108, {{0xc03c035710, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0567ff300, {{0xc03c035710, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05287e000, {0x63c85b8, 0xc02f62c870}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02f62c870}, {0x64a6560, 0xc0567ff300}, {0xc03c035710, 0xc00bc1fa40, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02f62c870}, {0x6466ed8, 0xc072e28100}, {0xc03c035710, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04318f638}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc066b42400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b11c00, {0x63c85b8, 0xc02f62c870}, 0xc066b42400, 0xc04328a3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b11c00, {0x63c85b8, 0xc02f62c870}, 0xc066b42400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f62c750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f62c750}, {0xc0394873f8, 0x660089}, 0xc039487400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f62c750}, {0x7fe3ac1576c8}, 0xc066b42100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f62c5a0}, 0xc066b42100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}, 0xc029cc36c0, 0xc01c3dfcf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}, 0x3, 0xc002de3880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc039487980}}}, {0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}, 0xc029cc36c0, 0xc002de3880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f62c5a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f62c5a0}, {0x40, 0xc039487b70}, 0xc039487ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f62c5a0}, {0x4e9d7a0, 0xc066b42100}, 0xc029cc36c0, 0xc01c3dfcf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f62c5a0}, 0xc06b883140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0206c8fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0206c8fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810337 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc06051ccd0, {0x63c85b8, 0xc077c27fb0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc077c27fb0}, {0x4f67ad2, 0x10beec0}, 0xc027908fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219804516 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc034ee76b0}, 0xc076342cf0, 0x1, 0xc05a447ff0, 0x519bbc, 0xc05f05a840, 0xc05eaf9020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc034ee76b0}, 0x120, 0xc05a448178, 0x5f05a7e0, 0x0, 0x660069, 0xc05f05a840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc034ee76b0}, 0xc05f05a7e0, {{{{0x0, 0x0}, {0xc05c3e0000, 0xf0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc034ee76b0}, 0x6352410, 0x421b0c28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc034ee76b0}, {0xc05f05a6c0, {0x17430110323939d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc034ee76b0}, 0xc071dac280, 0xc057c74938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc034ee76b0}, 0x660089, {0xc05f05a6c0, {0x17430110323939d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc034ee76b0}, 0xc046c98c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110323939d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110323939d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc034ee7500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc034ee7500}, {0xc057c76340, 0x660089}, 0xc057c76348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc034ee7500}, {0xc00102e4e0}, 0xc046c98a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc034ee74a0}, 0xc046c98a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc034ee6f60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219721648 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc06d1b88d0, 0x1f) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0404fca80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0404fd0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc07181d900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc07181d900, {0x63c8510, 0xc06d1b8900}, 0xc06f23fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc045626410) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06d1b8900}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc045626410) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219790450 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc06e51b210, 0x85) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc068502a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0685030e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc054722c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc054722c00, {0x63c8510, 0xc06e51b240}, 0xc08da99dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc028d40720) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06e51b240}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc028d40720) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809023 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc01e0ad200}, 0xc037e66750, 0x1, 0xc00df710a8, 0x519bbc, 0xc04c4eb6e0, 0xc03a341b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01e0ad200}, 0x120, 0xc00df71230, 0x4c4eb680, 0x0, 0x660069, 0xc04c4eb6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc01e0ad200}, 0xc04c4eb680, {{{{0x0, 0x0}, {0xc073d510e0, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01e0ad200}, 0x6352410, 0x300d02f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc01e0ad200}, {0xc04c4eb560, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc01e0ad200}, 0xc02e9028c0, 0xc00df719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01e0ad200}, 0x660089, {0xc04c4eb560, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc01e0ad200}, 0xc044eb7c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e0ad170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e0ad170}, {0xc00df733f8, 0x660089}, 0xc00df73400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e0ad170}, {0x126b7a013b010000}, 0xc044eb7900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e0ad0e0}, 0xc044eb7900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}, 0xc01c862ce0, 0xc059579320) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}, 0x3, 0xc006588b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00df73980}}}, {0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}, 0xc01c862ce0, 0xc006588b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01e0ad0e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e0ad0e0}, {0x40, 0xc00df73b70}, 0xc00df73ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01e0ad0e0}, {0x4e9d7a0, 0xc044eb7900}, 0xc01c862ce0, 0xc059579320) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01e0ad0e0}, 0xc02d0d75c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04c4eb440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04c4eb440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810971 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021c5a640, {0x63c85b8, 0xc037f8c5d0}, 0xc08b140c90, 0x1, 0xc01bc3d0a8, 0x519bbc, 0xc039319040, 0xc054c93f20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc049db8480, {0x63c85b8, 0xc037f8c5d0}, 0xc049db84b0, 0xc01bc3d1a8, 0x30, 0x40, 0xffffffffffffffff, 0xc039319040, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021c5a640, {0x63c85b8, 0xc037f8c5d0}, 0xc039318e00, {{{{0x0, 0x0}, {0xc05eecb7a0, 0x3}}, {{0xc067a5dd40, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc037f8c5d0}, 0x63521a0, 0x7b229cd8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c42640, {0x63c85b8, 0xc037f8c5d0}, {0xc02b1306c0, {0x1743010ff63ff233, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c42640, {0x63c85b8, 0xc037f8c5d0}, 0xc04dab9040, 0xc01bc3d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc037f8c5d0}, 0x660089, {0xc02b1306c0, {0x1743010ff63ff233, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c6b000, {0x63c85b8, 0xc037f8c5d0}, 0xc083804500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff63ff233, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff63ff233, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc037f8c510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037f8c510}, {0xc01bc3f3f8, 0x660089}, 0xc01bc3f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc037f8c510}, {0x40e427ffce0760fc}, 0xc083804300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc037f8c450}, 0xc083804300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}, 0xc044e81340, 0xc02f14d9e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}, 0x3, 0xc010094340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01bc3f980}}}, {0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}, 0xc044e81340, 0xc010094340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc037f8c450}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037f8c450}, {0x40, 0xc01bc3fb70}, 0xc01bc3fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc037f8c450}, {0x4e9d7a0, 0xc083804300}, 0xc044e81340, 0xc02f14d9e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc037f8c450}, 0xc014080a20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02b1305a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02b1305a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805370 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc04cd0d620}, 0xc025da19b0, 0x1, 0xc08c4730a8, 0x519bbc, 0xc05add0960, 0xc05b64c840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04cd0d620}, 0x120, 0xc08c473230, 0x5add0900, 0xc0506961a0, 0x18, 0xc05add0960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc04cd0d620}, 0xc05add0900, {{{{0x0, 0x0}, {0xc04fbe9d40, 0x1c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04cd0d620}, 0x6352410, 0x71c3f240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc04cd0d620}, {0xc05add07e0, {0x1743011041ec4a1c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc04cd0d620}, 0xc06fd99900, 0xc08c4739f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04cd0d620}, 0x660089, {0xc05add07e0, {0x1743011041ec4a1c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc04cd0d620}, 0xc0ab174c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041ec4a1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041ec4a1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04cd0d590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cd0d590}, {0xc08c4753f8, 0x660089}, 0xc08c475400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04cd0d590}, {0x4677c5}, 0xc0ab174a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04cd0d4d0}, 0xc0ab174a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}, 0xc051693b80, 0xc025da1f08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}, 0x3, 0xc013259480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc08c475980}}}, {0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}, 0xc051693b80, 0xc013259480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04cd0d4d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cd0d4d0}, {0x40, 0xc08c475b70}, 0xc08c475ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}, 0x203004, 0x203004) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04cd0d4d0}, {0x4e9d7a0, 0xc0ab174a00}, 0xc051693b80, 0xc025da1f08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04cd0d4d0}, 0xc0183b94a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05bac87e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05bac87e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802794 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01dd05000, {0x63c85b8, 0xc047e32de0}, 0xc046fdc700, 0xc06f966000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc047e32de0}, 0xc046fdc700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110255a3b77, 0x0, 0x0}, 0xc03060b020, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110255a3b77, 0x0, 0x0}, 0xc03060b020, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e32d50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e32d50}, {0xc062c413f8, 0x660089}, 0xc062c41400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e32d50}, {0x7fe338117418}, 0xc046fdc500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e32cc0}, 0xc046fdc500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}, 0xc0968d0720, 0xc06316fc08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}, 0x3, 0xc056d175c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062c41980}}}, {0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}, 0xc0968d0720, 0xc056d175c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047e32cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e32cc0}, {0x40, 0xc062c41b70}, 0xc062c41ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047e32cc0}, {0x4e9d7a0, 0xc046fdc500}, 0xc0968d0720, 0xc06316fc08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047e32cc0}, 0xc0974e23c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0486790e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0486790e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219797466 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa8c50, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0687af480, 0xc010db1000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0687af480, {0xc010db1000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0687af480, {0xc010db1000, 0x735, 0xc045252140}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0534f2cf0, {0xc010db1000, 0x7eda79, 0xc010db1005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc040317860, {0xc010db1000, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02d7d3818, {0xc010db1000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03f3e1078, {0x631b900, 0xc02d7d3818}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03f3e0e00, {0x7fe42ab59310, 0xc040317860}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03f3e0e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03f3e0e00, {0xc0712f9000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03ac1cfd8, {0xc0712f9000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06d19e088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06d19e088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06d19e660, {0x634aac0, 0xc06d19e088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06d19e000, 0xc044b7ba10, 0xc077dc742b, 0xc077dc742d, {0x635bec0, 0xc00ca8c1b0}, 0xc000d6ae00, 0xc0383da248, 0xc077dc7429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06d19e000, {0x63c85b8, 0xc06de453b0}, 0xc044b7ba00, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc093ccd04a, 0x8}}, 0x0, 0xc06de45350, 0xc06de45380, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06de453b0}, {0x643d810, 0xc040317860}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04c387dd0}, {0x643d810, 0xc040317860}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04c387dd0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 202012875 [select]: google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc030d5acd0, {0xc025b512d0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:179 +0x96 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc030d5acd0, {0xc025b512d0, 0xc0604106f0, 0xc0918daa68}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:173 +0x16f google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc027c44b10, {0xc025b512d0, 0xc0918daae0, 0xa85507}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:485 +0x32 io.ReadAtLeast({0x6328620, 0xc027c44b10}, {0xc025b512d0, 0x5, 0x5}, 0x5) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 google.golang.org/grpc/internal/transport.(*Stream).Read(0xc01aec0120, {0xc025b512d0, 0x5, 0x5}) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:469 +0xa5 google.golang.org/grpc.(*parser).recvMsg(0xc025b512c0, 0x7fffffff) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:559 +0x47 google.golang.org/grpc.recvAndDecompress(0xc0918dac60, 0xc01aec0120, {0x0, 0x0}, 0x7fffffff, 0xc0918dad90, {0x6367fb0, 0x9bf6f48}) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:690 +0x66 google.golang.org/grpc.recv(0xc0918dad08, {0x7fe42ab594c0, 0xc000808f40}, 0x0, {0x0, 0x0}, {0x4cfe5a0, 0xc0b27aa690}, 0xc0082db200, 0xc0918dad90, ...) google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:756 +0x6e google.golang.org/grpc.(*serverStream).RecvMsg(0xc036234c30, {0x4cfe5a0, 0xc0b27aa690}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1583 +0x185 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc040bc63e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:189 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:98 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x643dc30, 0xc040bc63e0}, 0xc06d11b5b0, {0x63c85b8, 0xc027c44d50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:361 +0xb9 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x63c85b8, 0xc027c44d50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:397 +0x45 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809486 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc00aa61470}, 0xc02c5eca08, 0x1, 0xc01a1c70a8, 0x519bbc, 0xc0879003c0, 0xc042e5fa40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc00aa61470}, 0x120, 0xc01a1c7230, 0x87900360, 0x0, 0x660069, 0xc0879003c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc00aa61470}, 0xc087900360, {{{{0x0, 0x0}, {0xc0552a18c0, 0x45}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc00aa61470}, 0x6352410, 0x46e777b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc00aa61470}, {0xc087900240, {0x1743011064fdff23, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc00aa61470}, 0xc05ed406e0, 0xc01a1c79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc00aa61470}, 0x660089, {0xc087900240, {0x1743011064fdff23, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc00aa61470}, 0xc0933c2d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011064fdff23, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011064fdff23, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc00aa61200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00aa61200}, {0xc01a1c93f8, 0x660089}, 0xc01a1c9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc00aa61200}, {0x7fe3c2fde4f0}, 0xc0933c2b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc00aa61080}, 0xc0933c2b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}, 0xc047483860, 0xc0c884d260) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}, 0x3, 0xc01b30ef80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a1c9980}}}, {0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}, 0xc047483860, 0xc01b30ef80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc00aa61080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00aa61080}, {0x40, 0xc01a1c9b70}, 0xc01a1c9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc00aa61080}, {0x4e9d7a0, 0xc0933c2b00}, 0xc047483860, 0xc0c884d260) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc00aa61080}, 0xc074752000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc087900120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc087900120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219681435 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc06e06bb50, 0x13) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0c9a58000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0c9a58660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc066225900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc066225900, {0x63c8510, 0xc06e06bb80}, 0xc090ac1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08c6a8ae0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06e06bb80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08c6a8ae0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809146 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc06b0a4b10}, 0xc056517620, 0x1, 0xc0157d50a8, 0x519bbc, 0xc070c3ff00, 0xc0702e8540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc03c81f3e0, {0x63c85b8, 0xc06b0a4b10}, 0xc03e484ea0, 0xc0157d51a8, 0x70c3ff00, 0x0, 0x1, 0xc070c3ff00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc06b0a4b10}, 0xc01de8e5a0, {{{{0xc05c03a6c0, 0x3e}, {0xc0786d0480, 0x52}}, {{0xc042891b00, 0x7b}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06b0a4b10}, 0x63521a0, 0x29b7a9e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc06b0a4b10}, {0xc08d27f680, {0x1743010fffc2b2ab, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc06b0a4b10}, 0xc0123ea140, 0xc0157d59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b0a4b10}, 0x660089, {0xc08d27f680, {0x1743010fffc2b2ab, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc06b0a4b10}, 0xc043615000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fffc2b2ab, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fffc2b2ab, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b0a4a80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b0a4a80}, {0xc0157d73f8, 0x660089}, 0xc0157d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b0a4a80}, {0x0}, 0xc043614d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b0a48a0}, 0xc043614d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}, 0xc03c1d0b20, 0xc05f433098) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}, 0x3, 0xc027eeb740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0157d7980}}}, {0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}, 0xc03c1d0b20, 0xc027eeb740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b0a48a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b0a48a0}, {0x40, 0xc0157d7b70}, 0xc0157d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b0a48a0}, {0x4e9d7a0, 0xc043614d00}, 0xc03c1d0b20, 0xc05f433098) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b0a48a0}, 0xc01de8e360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08d27f560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08d27f560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803870 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc01dafe360}, 0xc0297bf7a0, 0x1, 0x0, 0x1, 0xc04a785140, 0xc04d664060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc01dafe360}, 0x1e0, 0xc0781ea728, 0x4a7850e0, 0x0, 0x660069, 0xc04a785140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc01dafe360}, 0xc04a7850e0, {{{{0x0, 0x0}, {0xc025b2d8c0, 0x52}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc01dafe360}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc01dafe360}, {0xc055ee27e0, {0x1743011032129bc8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc01dafe360}, 0xc066c93a40, 0xc076d8cee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01dafe360}, 0x660089, {0xc055ee27e0, {0x1743011032129bc8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc01dafe360}, 0xc06f9e5300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032129bc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032129bc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01dafe2d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01dafe2d0}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01dafe2d0}, {0xc00102e4e0}, 0xc06f9e5200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01dafe270}, 0xc06f9e5200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07d081dd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807458 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4000, {0x63c85b8, 0xc03c984db0}, 0xc08d81fe00, 0xc0753de640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4000, {0x63c85b8, 0xc03c984db0}, 0xc08d81fe00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110575b8c65, 0x0, 0x0}, 0xc090a3e2f0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110575b8c65, 0x0, 0x0}, 0xc090a3e2f0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c984c90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c984c90}, {0xc0040a93f8, 0x660089}, 0xc0040a9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c984c90}, {0x0}, 0xc08d81fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c984ba0}, 0xc08d81fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}, 0xc03d0641e0, 0xc0580702a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}, 0x3, 0xc039983bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0040a9980}}}, {0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}, 0xc03d0641e0, 0xc039983bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c984ba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c984ba0}, {0x40, 0xc0040a9b70}, 0xc0040a9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c984ba0}, {0x4e9d7a0, 0xc08d81fc00}, 0xc03d0641e0, 0xc0580702a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c984ba0}, 0xc023673380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07c9aed80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07c9aed80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803689 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f6ec400, {0x63c85b8, 0xc044ceabd0}, 0xc08af94600, 0xc04eeaa5a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f6ec400, {0x63c85b8, 0xc044ceabd0}, 0xc08af94600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e4959b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e4959b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044ceab40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044ceab40}, {0xc075b7d3f8, 0x660089}, 0xc075b7d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044ceab40}, {0xc07c93f580}, 0xc08af94400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044ceaa80}, 0xc08af94400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}, 0xc0523172c0, 0xc0aba0c198) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}, 0x3, 0xc0247b5640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075b7d980}}}, {0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}, 0xc0523172c0, 0xc0247b5640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044ceaa80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044ceaa80}, {0x40, 0xc075b7db70}, 0xc075b7dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044ceaa80}, {0x4e9d7a0, 0xc08af94400}, 0xc0523172c0, 0xc0aba0c198) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044ceaa80}, 0xc059458d20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04012dc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04012dc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219792515 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc023fe06c0}, 0xc024603cb0, {0x646cc48, 0xc023fe07e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc023fe06c0}, 0xedb7a834e, 0xc0711106c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc023fe06c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0290a2240}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc024603ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0290a2240}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219743141 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe79b88, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc060f0bc00, 0xc066068000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc060f0bc00, {0xc066068000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc060f0bc00, {0xc066068000, 0x735, 0xc024b5aa00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc045159618, {0xc066068000, 0x7eda79, 0xc066068005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc026aadbc0, {0xc066068000, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc047919500, {0xc066068000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc064fcd3f8, {0x631b900, 0xc047919500}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc064fcd180, {0x7fe42ab59310, 0xc026aadbc0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc064fcd180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc064fcd180, {0xc038a1e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc015796dc8, {0xc038a1e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0e86f2b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0e86f2b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0e86f30e0, {0x634aac0, 0xc0e86f2b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0e86f2a80, 0xc076a17160, 0xc08cb9b42b, 0xc08cb9b42d, {0x635bec0, 0xc0610e0090}, 0xc000d6ae00, 0xc0425b9498, 0xc08cb9b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0e86f2a80, {0x63c85b8, 0xc045725950}, 0xc076a17150, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0ac804047, 0x8}}, 0x0, 0xc0457258f0, 0xc045725920, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc045725950}, {0x643d810, 0xc026aadbc0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0264253b0}, {0x643d810, 0xc026aadbc0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0264253b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802793 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024e79c00, {0x63c85b8, 0xc011292c90}, 0xc03285ea00, 0xc0a38f41e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024e79c00, {0x63c85b8, 0xc011292c90}, 0xc03285ea00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110255a1580, 0x0, 0x0}, 0xc03ba348c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110255a1580, 0x0, 0x0}, 0xc03ba348c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc011292b40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc011292b40}, {0xc05ff453f8, 0x660089}, 0xc05ff45400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc011292b40}, {0x7fe429e58510}, 0xc03285e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc011292a20}, 0xc03285e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}, 0xc068275c20, 0xc01d516138) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}, 0x3, 0xc0d43efd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05ff45980}}}, {0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}, 0xc068275c20, 0xc0d43efd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc011292a20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc011292a20}, {0x40, 0xc05ff45b70}, 0xc05ff45ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc011292a20}, {0x4e9d7a0, 0xc03285e800}, 0xc068275c20, 0xc01d516138) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc011292a20}, 0xc0543f51a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc048678fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc048678fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812558 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc04f5ed8c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0524ae000, 0xc0372b9168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04f5ed8c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0524ae000, {0xc0939a0d80, 0x29, 0x73}, {0xc07ae1edc0, 0x29, 0x4a}, 0x0, 0xd6) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07a75d538, {0xc0939a0d80, 0x1, 0x4a}, {0xc07ae1edc0, 0xc07a75c500, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07a75d538, {0xc0939a0d80, 0x29, 0x73}, {0xc07ae1edc0, 0x29, 0x4a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07a75c7c0, {0xc07ae1edc0, 0x0, 0xc0372b94e0}, 0x121ea30, 0x50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07a75c7c0, {0xc0939a0d80, 0x29, 0x4a}, {0xc07ae1edc0, 0x28, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07a75c500, {0xc07ae1edc0, 0x29, 0x4a}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0759a4008, {{0xc03a992d80, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04da76d00, {{0xc03a992d80, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0c0210400, {0x63c85b8, 0xc056615260}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056615260}, {0x64a6560, 0xc04da76d00}, {0xc03a992d80, 0xc0066984e0, 0x30}, {0x1743011094e4eabc, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056615260}, {0x6466ed8, 0xc0759a4000}, {0xc03a992d80, 0x28, 0x30}, {0x1743011094e4eabc, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01b19f6c8}, {{0x1743011094e4eabc, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011094e4eabc, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046448700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b2b24fbc, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0312b0400, {0x63c85b8, 0xc056615260}, 0xc046448700, 0xc01dbfbe00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0312b0400, {0x63c85b8, 0xc056615260}, 0xc046448700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011094e4eabc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011094e4eabc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0566151d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0566151d0}, {0xc0372bf3f8, 0x660089}, 0xc0372bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0566151d0}, {0x7fe428650748}, 0xc046448500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0566150e0}, 0xc046448500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}, 0xc0ccf7fe60, 0xc038161f20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}, 0x3, 0xc071e55340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0372bf980}}}, {0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}, 0xc0ccf7fe60, 0xc071e55340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0566150e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0566150e0}, {0x40, 0xc0372bfb70}, 0xc0372bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0566150e0}, {0x4e9d7a0, 0xc046448500}, 0xc0ccf7fe60, 0xc038161f20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0566150e0}, 0xc0719835c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea30e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea30e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800950 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc090ec0a80}, 0xc06b664c78, 0x1, 0xc04cb490a8, 0x519bbc, 0xc02d14aa20, 0xc014f1b5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06d45f620, {0x63c85b8, 0xc090ec0a80}, 0xc07ca00c60, 0xc04cb491a8, 0x2d14aa20, 0x0, 0x0, 0xc02d14aa20, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc090ec0a80}, 0xc02d14a960, {{{{0xc07a1cd680, 0x1d}, {0xc042bb6900, 0x37}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x30, {0x63c85b8, 0xc090ec0a80}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc090ec0a80}, {0xc05c0a2000, {0x17430110163de951, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc090ec0a80}, 0xc05f31cc80, 0xc04cb499f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc090ec0a80}, 0x660089, {0xc05c0a2000, {0x17430110163de951, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc090ec0a80}, 0xc05a917500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110163de951, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110163de951, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc090ec09f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090ec09f0}, {0xc04cb4b3f8, 0x660089}, 0xc04cb4b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc090ec09f0}, {0xbebf83c42872616d}, 0xc05a917300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc090ec0960}, 0xc05a917300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}, 0xc06ef559a0, 0xc00de340f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}, 0x3, 0xc05b781c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04cb4b980}}}, {0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}, 0xc06ef559a0, 0xc05b781c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc090ec0960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090ec0960}, {0x40, 0xc04cb4bb70}, 0xc04cb4bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc090ec0960}, {0x4e9d7a0, 0xc05a917300}, 0xc06ef559a0, 0xc00de340f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc090ec0960}, 0xc078d3e840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc038d4bc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc038d4bc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812639 [runnable]: internal/poll.runtime_pollWait(0x7fdbafee9e20, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06d853c80, 0xc0a57e6480, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06d853c80, {0xc0a57e6480, 0x205, 0x205}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06d853c80, {0xc0a57e6480, 0xc022290ab0, 0x66046f}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc033a92d98, {0xc0a57e6480, 0x0, 0x205}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04d73b080, {0xc0a57e6480, 0x2, 0x205}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04a6d9c20, {0xc0a57e6480, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0787b2978, {0x631b900, 0xc04a6d9c20}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0787b2700, {0x7fe42ab59310, 0xc04d73b080}, 0xcc) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0787b2700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).readHandshake(0xc0787b2700) GOROOT/src/crypto/tls/conn.go:1010 +0x6d crypto/tls.(*Conn).readClientHello(0xc0787b2700, {0x63c8510, 0xc050a07300}) GOROOT/src/crypto/tls/handshake_server.go:132 +0x45 crypto/tls.(*Conn).serverHandshake(0xc0787b2700, {0x63c8510, 0xc050a07300}) GOROOT/src/crypto/tls/handshake_server.go:43 +0x46 crypto/tls.(*Conn).handshakeContext(0xc0787b2700, {0x63c8548, 0xc000140068}) GOROOT/src/crypto/tls/conn.go:1453 +0x36f crypto/tls.(*Conn).HandshakeContext(...) GOROOT/src/crypto/tls/conn.go:1396 crypto/tls.(*Conn).Handshake(...) GOROOT/src/crypto/tls/conn.go:1380 crypto/tls.(*Conn).Read(0xc0787b2700, {0xc073d14a98, 0x4, 0xc019549520}) GOROOT/src/crypto/tls/conn.go:1265 +0x65 io.ReadAtLeast({0x7fe456a34450, 0xc0787b2700}, {0xc073d14a98, 0x4, 0x4}, 0x4) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadUntypedMsg(0xc073d14a80, {0x7fe456a34450, 0xc0787b2700}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:151 +0x65 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).readVersion(0xc00c689e00, {0x7fe456a34450, 0xc0787b2700}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1357 +0x11f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).maybeUpgradeToSecureConn(0xc00c689e00, {0x0, 0xffc}, {0x643d810, 0xc04d73b080}, 0x2, 0x4d2162f, 0xc09365bbe0) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1301 +0x367 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c8510, 0xc050a072c0}, {0x643d810, 0xc04d73b080}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:807 +0x5a9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc073d149c0}, {0x643d810, 0xc04d73b080}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc073d149c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 210818948 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc06e51afd0, 0x2245) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc005dc2a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc005dc30e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc060d0d300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc060d0d300, {0x63c8510, 0xc06e51b000}, 0xc05186fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc049b839c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06e51b000}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc049b839c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812320 [chan receive]: github.com/cockroachdb/pebble/sstable.(*writeQueue).runWorker(0xc0286f2a40) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:95 +0x4c created by github.com/cockroachdb/pebble/sstable.newWriteQueue github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/write_queue.go:50 +0xdb goroutine 219803424 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ee1f000, {0x63c85b8, 0xc06eac5380}, 0xc04366e000, 0xc051ff06e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ee1f000, {0x63c85b8, 0xc06eac5380}, 0xc04366e000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e8f84eb, 0x0, 0x0}, 0xc046af1950, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e8f84eb, 0x0, 0x0}, 0xc046af1950, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06eac52c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06eac52c0}, {0xc076857918, 0x660089}, 0xc076857920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06eac52c0}, {0xc00102e4e0}, 0xc0d503ff00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06eac5260}, 0xc0d503ff00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc03a6b90e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06eac51d0}, {0xc00dfd36b0, 0x24}, 0x46c7bf85f, 0xc0385e7d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc022fd2c30}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc06eac51d0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812046 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc028798930}, 0xc0ce7a4108, 0x1, 0x0, 0x1, 0xc085b9fec0, 0xc054507020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc028798930}, 0x120, 0xc0363a8728, 0x85b9fe60, 0x0, 0x660069, 0xc085b9fec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc028798930}, 0xc085b9fe60, {{{{0x0, 0x0}, {0xc033798240, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc028798930}, 0x6352410, 0x204d2428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc028798930}, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc028798930}, 0xc05b095400, 0xc07c880ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc028798930}, 0x660089, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc028798930}, 0xc060fbbb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0287988a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0287988a0}, {0xc0ad8368f0, 0x660089}, 0xc0ad8368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0287988a0}, {0xc00102e4e0}, 0xc060fbba00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc028798840}, 0xc060fbba00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0725420c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219769934 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea29b0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc066ce8780, 0xc0083de800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc066ce8780, {0xc0083de800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc066ce8780, {0xc0083de800, 0x735, 0xc06706f2c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0381d3c90, {0xc0083de800, 0x7eda79, 0xc0083de805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc08d0db800, {0xc0083de800, 0x76a2af81ede18848, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc05bec9200, {0xc0083de800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc072665af8, {0x631b900, 0xc05bec9200}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc072665880, {0x7fe42ab59310, 0xc08d0db800}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc072665880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc072665880, {0xc082c7b000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02d727a70, {0xc082c7b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0577db588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0577db588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0577dbb60, {0x634aac0, 0xc0577db588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0577db500, 0xc02bb2ef20, 0xc086abd42b, 0xc086abd42d, {0x635bec0, 0xc06cb4e7e0}, 0xc000d6ae00, 0xc08ccac898, 0xc086abd429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0577db500, {0x63c85b8, 0xc0ce085dd0}, 0xc02bb2ef10, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0687b1021, 0x8}}, 0x0, 0xc0ce085d70, 0xc0ce085da0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0ce085dd0}, {0x643d810, 0xc08d0db800}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0676f9680}, {0x643d810, 0xc08d0db800}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0676f9680}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790387 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc071a917a0}, 0xc024602cb0, {0x646cc48, 0xc071a91800}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc071a917a0}, 0xedb7a834e, 0xc079a86b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc071a917a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06c8d3c80}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc024602ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06c8d3c80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805766 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64000, {0x63c85b8, 0xc028491770}, 0xc041909b90, 0x1, 0xc005d530a8, 0x519bbc, 0xc08f9be600, 0xc058d0b380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc028491770}, 0x120, 0xc005d53230, 0x8f9be5a0, 0x0, 0x660069, 0xc08f9be600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64000, {0x63c85b8, 0xc028491770}, 0xc08f9be5a0, {{{{0x0, 0x0}, {0xc041f937a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc028491770}, 0x6352410, 0x52e791f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a588c0, {0x63c85b8, 0xc028491770}, {0xc08f9be480, {0x174301103f87867f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a588c0, {0x63c85b8, 0xc028491770}, 0xc050935180, 0xc005d539f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc028491770}, 0x660089, {0xc08f9be480, {0x174301103f87867f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016ba0000, {0x63c85b8, 0xc028491770}, 0xc046dbe100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0284916e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0284916e0}, {0xc005d553f8, 0x660089}, 0xc005d55400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0284916e0}, {0x7fe3921d0080}, 0xc09850bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc028491650}, 0xc09850bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}, 0xc03d586540, 0xc0799e3830) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}, 0x3, 0xc08ae71f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005d55980}}}, {0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}, 0xc03d586540, 0xc08ae71f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc028491650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028491650}, {0x40, 0xc005d55b70}, 0xc005d55ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc028491650}, {0x4e9d7a0, 0xc09850bf00}, 0xc03d586540, 0xc0799e3830) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc028491650}, 0xc031b41b60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03242e000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03242e000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809158 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0757bad80}, 0xc028b31cb0, {0x646cc48, 0xc0757bade0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0757bad80}, 0xedb7a8350, 0xc01e336c60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0757bad80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc062c31ec0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc028b31ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc062c31ec0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810015 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030720000, {0x63c85b8, 0xc05231c0c0}, 0xc049a6d600, 0xc05002a140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030720000, {0x63c85b8, 0xc05231c0c0}, 0xc049a6d600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106b8c2d93, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106b8c2d93, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05231c030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05231c030}, {0xc010d6f3f8, 0x660089}, 0xc010d6f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05231c030}, {0x210030803ea2202}, 0xc049a6d400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0561fdf80}, 0xc049a6d400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}, 0xc07dae9ac0, 0xc0b58e68e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}, 0x3, 0xc0ae217600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010d6f980}}}, {0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}, 0xc07dae9ac0, 0xc0ae217600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0561fdf80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0561fdf80}, {0x40, 0xc010d6fb70}, 0xc010d6fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0561fdf80}, {0x4e9d7a0, 0xc049a6d400}, 0xc07dae9ac0, 0xc0b58e68e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0561fdf80}, 0xc0b6af8540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04d2d1200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04d2d1200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811026 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc021401c20}, 0xc03c4f5c08, 0x1, 0xc083e7bf20, 0x519bbc, 0xc05cac45a0, 0xc05a936540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc021401c20}, 0xc0, 0xc083e7c0a8, 0x5cac4540, 0x0, 0x660069, 0xc05cac45a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc021401c20}, 0xc05cac4540, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc021401c20}, 0x63524a0, 0x412ad690) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc021401c20}, {0x0, {0x1743011071bc055c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc021401c20}, 0xc0601d9360, 0x1) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc021401c20}, 0x660089, {0x0, {0x1743011071bc055c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc021401c20}, 0xc05760e000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021401b90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021401b90}, {0xc00f304270, 0x660089}, 0xc00f304278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021401b90}, {0xc0214017d0}, 0xc0b3391f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021401b30}, 0xc0b3391f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x1743011071bc055c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc021401aa0}, 0x0, 0xc04f509b80) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc021401aa0}, 0x66040f) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc019919860, {0x63c85b8, 0xc021401aa0}, {{0xa, 0x4a, 0x6c, 0xd7, 0x50, 0x7f, 0x4f, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:922 +0x1b0 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc019919860, {0x63c85b8, 0xc021401aa0}, 0xc0422dfe00, 0xc0581f6520, 0xc04eee8c30) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:610 +0xd4a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc021401aa0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc021401a10}, {{0xc0479987c0, 0x24, 0x24}}, 0xc00f307380) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc019919860, {0x63c85b8, 0xc021401a10}, 0xc0422dfe00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0301af310, {0x63c85b8, 0xc021401a10}, {0x0, {0x1743011071b4498d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc021401a10}, 0xc04b6ad360, 0xc00f3079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc021401a10}, 0x660089, {0x0, {0x1743011071b4498d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc021401a10}, 0xc0b3391e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011071b4498d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011071b4498d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011071b4498d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021401890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021401890}, {0xc00f3093f8, 0x660089}, 0xc00f309400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021401890}, {0x0}, 0xc0b3391c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0214017d0}, 0xc0b3391c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}, 0xc0581f64e0, 0xc050e4e3c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}, 0x3, 0xc047998700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f309980}}}, {0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}, 0xc0581f64e0, 0xc047998700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0214017d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0214017d0}, {0x40, 0xc00f309b70}, 0xc00f309ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0214017d0}, {0x4e9d7a0, 0xc0b3391c00}, 0xc0581f64e0, 0xc050e4e3c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0214017d0}, 0xc044800120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05aed9b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05aed9b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801291 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc028d5e1e0}, 0xc00776ecb0, {0x646cc48, 0xc028d5e240}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc028d5e1e0}, 0xedb7a834f, 0xc04bed7c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc028d5e1e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc01620c3c0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc00776eef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc01620c3c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219757005 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc040433e40, {0x63c85b8, 0xc034ee6d80}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc04cac8c00, {0x63c85b8, 0xc034ee6d80}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc04cac8c00, {0x63c8510, 0xc0667f0c40}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc04cac8c00, {0x63c8510, 0xc0667f0c40}, {0x63f7db8, 0x9bf6f48}, 0xc08b2e5088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc04cac8c00, {0x63c85b8, 0xc00f4430b0}, {{0x63f7db8, 0x9bf6f48}, {0xc00ebe016e, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc04cac8c00, {0x63c85b8, 0xc00f4430b0}, {{0x63f7db8, 0x9bf6f48}, {0xc00ebe016e, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc00ebe016e, 0x6}, 0x0, 0x0}, {0x188ecf07, 0xedb7a834f, 0x0}, {0x188ed675, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04cac8c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04cac8c00, {0x63c8510, 0xc063e66900}, 0xc070133dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0435348d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc063e66900}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0435348d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812015 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ffd4000, {0x63c85b8, 0xc049633320}, 0xc026a0a588, 0x1, 0xc0386350a8, 0x519bbc, 0xc05153c960, 0xc09fd4c2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc049633320}, 0x120, 0xc038635230, 0x5153c900, 0x0, 0x660069, 0xc05153c960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ffd4000, {0x63c85b8, 0xc049633320}, 0xc05153c900, {{{{0x0, 0x0}, {0xc05ab877a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc049633320}, 0x6352410, 0x33a92690) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01f9d8c30, {0x63c85b8, 0xc049633320}, {0xc05153c7e0, {0x174301107dee08fa, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01f9d8c30, {0x63c85b8, 0xc049633320}, 0xc06bf1cc80, 0xc0386359f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc049633320}, 0x660089, {0xc05153c7e0, {0x174301107dee08fa, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01fab9400, {0x63c85b8, 0xc049633320}, 0xc04b09bb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107dee08fa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107dee08fa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc049633290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049633290}, {0xc0386373f8, 0x660089}, 0xc038637400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc049633290}, {0x7fe3b8b34988}, 0xc04b09b900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc049633200}, 0xc04b09b900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}, 0xc05bd8a240, 0xc026a0a7b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}, 0x3, 0xc041a0aa40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc038637980}}}, {0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}, 0xc05bd8a240, 0xc041a0aa40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc049633200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049633200}, {0x40, 0xc038637b70}, 0xc038637ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc049633200}, {0x4e9d7a0, 0xc04b09b900}, 0xc05bd8a240, 0xc026a0a7b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc049633200}, 0xc0ba0c8660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05153c6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05153c6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799807 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc06034de60}, 0xc01646ccb0, {0x646cc48, 0xc06034dec0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc06034de60}, 0xedb7a834e, 0xc073766480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc06034de60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc024e7d1d0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01646cef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc024e7d1d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808626 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc061dce000, {0x63c85b8, 0xc02d01aea0}, 0xc0c8bdbe00, 0xc03babe820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc02d01aea0}, 0xc0c8bdbe00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02d01ad80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d01ad80}, {0xc00d6233f8, 0x660089}, 0xc00d623400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02d01ad80}, {0x3feffff847505b38}, 0xc0c8bdbb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02d01ac00}, 0xc0c8bdbb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}, 0xc05159a380, 0xc03592c0f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}, 0x3, 0xc058e092c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00d623980}}}, {0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}, 0xc05159a380, 0xc058e092c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02d01ac00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d01ac00}, {0x40, 0xc00d623b70}, 0xc00d623ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02d01ac00}, {0x4e9d7a0, 0xc0c8bdbb00}, 0xc05159a380, 0xc03592c0f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02d01ac00}, 0xc02ac2ec00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ebc4480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ebc4480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810598 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc042f3d8c0}, 0xc0456ae1c8, 0x1, 0xc02bddf0a8, 0x519bbc, 0xc08511a960, 0xc081db7a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc042f3d8c0}, 0x120, 0xc02bddf230, 0x8511a900, 0x0, 0x660069, 0xc08511a960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc042f3d8c0}, 0xc08511a900, {{{{0x0, 0x0}, {0xc032b04fc0, 0x2b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc042f3d8c0}, 0x6352410, 0x1baa27f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc042f3d8c0}, {0xc08511a6c0, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc042f3d8c0}, 0xc0cc967d60, 0xc02bddf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc042f3d8c0}, 0x660089, {0xc08511a6c0, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc042f3d8c0}, 0xc03d7dcb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042f3d830}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042f3d830}, {0xc02bde13f8, 0x660089}, 0xc02bde1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042f3d830}, {0x7fe13d0759b8}, 0xc03d7dc900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042f3d7a0}, 0xc03d7dc900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}, 0xc064568700, 0xc08c1396b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}, 0x3, 0xc0a859d800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02bde1980}}}, {0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}, 0xc064568700, 0xc0a859d800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc042f3d7a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042f3d7a0}, {0x40, 0xc02bde1b70}, 0xc02bde1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc042f3d7a0}, {0x4e9d7a0, 0xc03d7dc900}, 0xc064568700, 0xc08c1396b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc042f3d7a0}, 0xc075145980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0457b4b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0457b4b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806903 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f22b2c0, {0x63c85b8, 0xc091fefc80}, 0xc02f43f1a0, 0x1, 0xc0037670a8, 0x519bbc, 0xc081f8ad20, 0xc017fef740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05b374d80, {0x63c85b8, 0xc091fefc80}, 0xc081f8ad20, 0xc0037671a8, 0x31, 0x40, 0x0, 0xc081f8ad20, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f22b2c0, {0x63c85b8, 0xc091fefc80}, 0xc081f8ac60, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04d8f6a20, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc091fefc80}, 0x6352260, 0x4f454b50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f219ae0, {0x63c85b8, 0xc091fefc80}, {0xc081f8ab40, {0x17430110173aa8b8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f219ae0, {0x63c85b8, 0xc091fefc80}, 0xc055072c80, 0xc0037679f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc091fefc80}, 0x660089, {0xc081f8ab40, {0x17430110173aa8b8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02072e800, {0x63c85b8, 0xc091fefc80}, 0xc07447ed00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110173aa8b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110173aa8b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc091fefbf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc091fefbf0}, {0xc0037693f8, 0x660089}, 0xc003769400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc091fefbf0}, {0xc03cf6b3f0}, 0xc07447ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc091fefb60}, 0xc07447ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}, 0xc08fc485e0, 0xc038af8d08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}, 0x3, 0xc09071c980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003769980}}}, {0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}, 0xc08fc485e0, 0xc09071c980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc091fefb60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc091fefb60}, {0x40, 0xc003769b70}, 0xc003769ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc091fefb60}, {0x4e9d7a0, 0xc07447ea00}, 0xc08fc485e0, 0xc038af8d08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc091fefb60}, 0xc049665f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc081f8aa20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc081f8aa20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812583 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807500 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc0571891d0}, 0xc03c049398, 0x1, 0xc00603b0a8, 0x519bbc, 0xc08e5a6cc0, 0xc08d13acc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0571891d0}, 0x120, 0xc00603b230, 0x8e5a6c60, 0x0, 0x660069, 0xc08e5a6cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc0571891d0}, 0xc08e5a6c60, {{{{0x0, 0x0}, {0xc06540e6c0, 0x10}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0571891d0}, 0x6352410, 0x23d19598) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc0571891d0}, {0xc08e5a6b40, {0x17430110576c2c1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc0571891d0}, 0xc0a38f4f00, 0xc00603b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0571891d0}, 0x660089, {0xc08e5a6b40, {0x17430110576c2c1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc0571891d0}, 0xc097bae200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110576c2c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110576c2c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057189110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057189110}, {0xc00603d3f8, 0x660089}, 0xc00603d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057189110}, {0x7fe42b75e0d8}, 0xc097bae000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057189080}, 0xc097bae000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}, 0xc020144000, 0xc015f71080) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}, 0x3, 0xc0cd3a74c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00603d980}}}, {0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}, 0xc020144000, 0xc0cd3a74c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc057189080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057189080}, {0x40, 0xc00603db08}, 0xc00603dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc057189080}, {0x4e9d7a0, 0xc097bae000}, 0xc020144000, 0xc015f71080) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc057189080}, 0xc04bb04720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08e5a6a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08e5a6a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791328 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc054c17080}, 0xc03b926600, 0xc047931f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc054c17080}, 0xc03b926600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc054c16ff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054c16ff0}, {0xc07bd427f8, 0x660089}, 0xc07bd42800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc054c16ff0}, {0x24}, 0xc01b942800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc054c16f90}, 0xc01b942800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc049f49d90, {0x63c85b8, 0xc054c16e10}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0acb40600, {0x63c85b8, 0xc054c16e10}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0acb40600, {0x63c8510, 0xc03faf5bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc0acb40600, {0x63c8510, 0xc03faf5bc0}, {0x63f7db8, 0x9bf6f48}, 0xc07bd47088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0acb40600, {0x63c85b8, 0xc02248e6c0}, {{0x63f7db8, 0x9bf6f48}, {0xc006b08a05, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0acb40600, {0x63c85b8, 0xc02248e6c0}, {{0x63f7db8, 0x9bf6f48}, {0xc006b08a05, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc006b08a05, 0x6}, 0x0, 0x0}, {0x376709b, 0xedb7a8350, 0x0}, {0x37677e8, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0acb40600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0acb40600, {0x63c8510, 0xc029351700}, 0xc075f49dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc00df8abb0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc029351700}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc00df8abb0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812773 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805458 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc06f159740}, 0xc032d85b48, 0x1, 0x708017dfda9200, 0x37e11d600, 0xc05bbd5140, 0xc0716aef00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06f159740}, 0x120, 0xc0978c5230, 0x5bbd50e0, 0xc03cc25380, 0x18, 0xc05bbd5140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc06f159740}, 0xc05bbd50e0, {{{{0x0, 0x0}, {0xc058fcd680, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06f159740}, 0x6352410, 0x6618a318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc06f159740}, {0xc05bbd4fc0, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc06f159740}, 0xc081cd4fa0, 0xc0978c59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f159740}, 0x660089, {0xc05bbd4fc0, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc06f159740}, 0xc059e96f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f159680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f159680}, {0xc0978c73f8, 0x660089}, 0xc0978c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f159680}, {0x0}, 0xc059e96c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f159560}, 0xc059e96c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}, 0xc00af91aa0, 0xc032d85b18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}, 0x3, 0xc06c1a4ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0978c7980}}}, {0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}, 0xc00af91aa0, 0xc06c1a4ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f159560}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f159560}, {0x40, 0xc0978c7b70}, 0xc0978c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f159560}, {0x4e9d7a0, 0xc059e96c00}, 0xc00af91aa0, 0xc032d85b18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f159560}, 0xc04967fb00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05bbd4ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05bbd4ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791184 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc036745ce0}, 0xc028b36cb0, {0x646cc48, 0xc036745d40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc036745ce0}, 0xedb7a834e, 0xc053008a20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc036745ce0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc049628bd0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc028b36ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc049628bd0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219795756 [runnable]: sync.runtime_SemacquireMutex(0xc0833b0690, 0x8f, 0x7fd4b7635ec0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0833b0538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0833b0560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04de04480, {0xffffffffffffff15, 0xc0833b06c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc04de04480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04de04480, {0xc03a0ae150, 0x32, 0x6f}, {0xc018019000, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0886b3e98, {0xc03a0ae150, 0x32, 0x6f}, {0xc018019000, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0886b27c0, {0xc018019000, 0x77, 0x6f}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0886b27c0, {0xc03a0ae150, 0x77, 0xc018019000}, {0xc018019000, 0xc072d06ea0, 0x31}, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0886b2500, {0xc018019000, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc09dea38a8, {{0xc072d06ea0, 0x31, 0xd0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04be2c000, {{0xc04a5e02d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc048dd2800, {0x63c85b8, 0xc0837a3680}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0837a3680}, {0x64a6560, 0xc04be2c000}, {0xc04a5e02d0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0837a3680}, {0x6466ed8, 0xc09dea3600}, {0xc04a5e02d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc041e5f860}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc055eea700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0201d8800, {0x63c85b8, 0xc0837a3680}, 0xc055eea700, 0xc07236b7c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0201d8800, {0x63c85b8, 0xc0837a3680}, 0xc055eea700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0837a35f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0837a35f0}, {0xc0110d48f0, 0x660089}, 0xc0110d48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0837a35f0}, {0xc00102e4e0}, 0xc055eea400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0837a3590}, 0xc055eea400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0837a32c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802577 [runnable]: sync.runtime_SemacquireMutex(0xc0912ea690, 0x8f, 0x7fe187f0c300) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x1194e0b, 0xc0912ea4f0, 0x11948b8) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x68, 0x32, 0x69, 0x32) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc041dba900, {0xffffffffffffff15, 0xc0912ea6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc041dba900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc041dba900, {0xc0ab918900, 0x32, 0x71}, {0xc024b7fec0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092ef5e98, {0xc0ab918900, 0x32, 0x71}, {0xc024b7fec0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092ef47c0, {0xc024b7fec0, 0x32, 0x71}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092ef47c0, {0xc0ab918900, 0x32, 0xc024b7fec0}, {0xc024b7fec0, 0xc053878f00, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092ef4500, {0xc024b7fec0, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc065dda828, {{0xc053878f00, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc09c55e700, {{0xc0603fe810, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc047862400, {0x63c85b8, 0xc07cbfe7b0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc07cbfe7b0}, {0x64a6560, 0xc09c55e700}, {0xc0603fe810, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc07cbfe7b0}, {0x6466ed8, 0xc065dda580}, {0xc0603fe810, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0770d82b8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc02754d400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4ac00, {0x63c85b8, 0xc07cbfe7b0}, 0xc02754d400, 0xc061f765a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4ac00, {0x63c85b8, 0xc07cbfe7b0}, 0xc02754d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07cbfe720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07cbfe720}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07cbfe720}, {0xc00102e4e0}, 0xc02754d200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07cbfe6c0}, 0xc02754d200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220c30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802533 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0450a7bf0}, 0xc0ce426690, 0x1, 0xc055c17ff0, 0x519bbc, 0xc03ad21500, 0xc03afc0d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0450a7bf0}, 0x1e0, 0xc055c18178, 0x3ad214a0, 0x0, 0x660069, 0xc03ad21500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0450a7bf0}, 0xc03ad214a0, {{{{0x0, 0x0}, {0xc0031d26c0, 0xc8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0450a7bf0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0450a7bf0}, {0xc06062c000, {0x17430110234b556f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0450a7bf0}, 0xc0726c7900, 0xc076588938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0450a7bf0}, 0x660089, {0xc06062c000, {0x17430110234b556f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0450a7bf0}, 0xc011fdcc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110234b556f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110234b556f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0450a7ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0450a7ad0}, {0xc07658a340, 0x660089}, 0xc07658a348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0450a7ad0}, {0xc00102e4e0}, 0xc011fdcb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0450a7a70}, 0xc011fdcb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0479fae10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812660 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811691 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc066631ad0}, 0xc02902c0a8, 0x1, 0xc0393d30a8, 0x519bbc, 0xc0778ffb60, 0xc00f62e2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc066631ad0}, 0x120, 0xc0393d3230, 0x778ffb00, 0x0, 0x660069, 0xc0778ffb60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc066631ad0}, 0xc0778ffb00, {{{{0x0, 0x0}, {0xc04fee4d80, 0x120}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc066631ad0}, 0x6352410, 0x36ce80c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc066631ad0}, {0xc0778ff9e0, {0x174301107f0ff4ae, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc066631ad0}, 0xc062113400, 0xc0393d39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066631ad0}, 0x660089, {0xc0778ff9e0, {0x174301107f0ff4ae, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc066631ad0}, 0xc04fb2c900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f0ff4ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f0ff4ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066631a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066631a40}, {0xc0393d53f8, 0x660089}, 0xc0393d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066631a40}, {0x7fe3921c8e50}, 0xc04fb2c700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0666319b0}, 0xc04fb2c700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}, 0xc0405015c0, 0xc02902c288) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}, 0x3, 0xc06aff34c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0393d5980}}}, {0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}, 0xc0405015c0, 0xc06aff34c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0666319b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0666319b0}, {0x40, 0xc0393d5b70}, 0xc0393d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0666319b0}, {0x4e9d7a0, 0xc04fb2c700}, 0xc0405015c0, 0xc02902c288) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0666319b0}, 0xc07d452180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0778ff8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0778ff8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219789914 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc079205560}, 0xc04a611cb0, {0x646cc48, 0xc0792055c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc079205560}, 0xedb7a834d, 0xc0763ef200) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc079205560}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0238cbf50}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc04a611ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0238cbf50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812621 [runnable]: sync.runtime_SemacquireMutex(0xc042f94e38, 0xcd, 0xc00074a760) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0x118663c, 0xc042f94ea0, 0x11844cd) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc042f94f98, 0x11a5314, 0x7fe2824f4140, 0x7fdd9a5fc0c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x4d866e, 0x9185600}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).init(0xc04ea1e000, 0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1229 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2157 +0xdb github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a528c0, 0xc042f95188, 0xc08967fee8, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc089680000, 0x0, 0xc089680000) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc08967fe98, 0xc0734106e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc08967fe98, {0xc0a090eb80, 0x1e, 0x72}, {0xc0128fce80, 0x1e, 0x72}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc08967e7c0, {0xc0128fce80, 0x2, 0x4}, 0x1b, 0xc8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc08967e7c0, {0xc0a090eb80, 0x72, 0xc04dcdb770}, {0xc0128fce80, 0xc04dcdb770, 0x1d}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc08967e500, {0xc0128fce80, 0x1e, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06f224ea8, {{0xc04dcdb770, 0x1d, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc040011100, {{0xc08467aae0, 0x16, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc045cf3400, {0x63c85b8, 0xc02f2c4210}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02f2c4210}, {0x64a6560, 0xc040011100}, {0xc08467aae0, 0x40c48eff8a084318, 0x18}, {0x1743011097ee4abf, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02f2c4210}, {0x6466ed8, 0xc06f224c00}, {0xc08467aae0, 0x16, 0x18}, {0x1743011097ee4abf, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc021e7ca38}, {{0x1743011097ee4abf, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011097ee4abf, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0460dfc00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b5bbafbf, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0185c3800, {0x63c85b8, 0xc02f2c4210}, 0xc0460dfc00, 0xc051998b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0185c3800, {0x63c85b8, 0xc02f2c4210}, 0xc0460dfc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011097ee4abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011097ee4abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f2c4180}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f2c4180}, {0xc042f9b3f8, 0x660089}, 0xc042f9b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f2c4180}, {0x4053ff89bbebb52c}, 0xc0460dfa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0304cbf80}, 0xc0460dfa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}, 0xc03ebdd9e0, 0xc031cfb6c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}, 0x3, 0xc015b9e980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042f9b980}}}, {0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}, 0xc03ebdd9e0, 0xc015b9e980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0304cbf80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304cbf80}, {0x40, 0xc042f9bb70}, 0xc042f9bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0304cbf80}, {0x4e9d7a0, 0xc0460dfa00}, 0xc03ebdd9e0, 0xc031cfb6c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0304cbf80}, 0xc01c3423c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356ab40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356ab40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812698 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805968 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc031b01e60}, 0xc032b1fcb0, {0x646cc48, 0xc031b01ec0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc031b01e60}, 0xedb7a834f, 0xc00e65b7a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc031b01e60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc074b11e60}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc032b1fef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc074b11e60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219795858 [runnable]: sync.runtime_SemacquireMutex(0xc05fa7c690, 0x8f, 0x7fe227372300) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc05fa7c538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc05fa7c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc065330000, {0xffffffffffffff15, 0xc05fa7c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc065330000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc065330000, {0xc02f108600, 0x32, 0x72}, {0xc06e5b3b00, 0x32, 0x89}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07cd70398, {0xc02f108600, 0x32, 0x72}, {0xc06e5b3b00, 0x32, 0x89}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07cd6ecc0, {0xc06e5b3b00, 0x89, 0x72}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07cd6ecc0, {0xc02f108600, 0x89, 0xc06e5b3b00}, {0xc06e5b3b00, 0xc05044d180, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07cd6ea00, {0xc06e5b3b00, 0x32, 0x89}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc059836ea8, {{0xc05044d180, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc09fe45700, {{0xc01f715c20, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc053cb8000, {0x63c85b8, 0xc0837a26f0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0837a26f0}, {0x64a6560, 0xc09fe45700}, {0xc01f715c20, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0837a26f0}, {0x6466ed8, 0xc059836c00}, {0xc01f715c20, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0282d0f00}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc077208800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0201d4c00, {0x63c85b8, 0xc0837a26f0}, 0xc077208800, 0xc071c065a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0201d4c00, {0x63c85b8, 0xc0837a26f0}, 0xc077208800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0837a2660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0837a2660}, {0xc09365e8f0, 0x660089}, 0xc09365e8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0837a2660}, {0xc00102e4e0}, 0xc077208700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0837a2600}, 0xc077208700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc062c8ac00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812834 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc04f5ed8c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc041995000, 0xc0532f8700, 0xc041995168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04f5ed8c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0532f8700, {0xc023f77000, 0x29, 0x73}, {0xc0295f4640, 0x29, 0x4b}, 0x0, 0x4a) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0931a4938, {0xc023f77000, 0x1, 0x4b}, {0xc0295f4640, 0xc0931a3900, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0931a4938, {0xc023f77000, 0x29, 0x73}, {0xc0295f4640, 0x29, 0x4b}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0931a3bc0, {0xc0295f4640, 0x0, 0xc0419954e0}, 0x121ea30, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0931a3bc0, {0xc023f77000, 0x29, 0x4b}, {0xc0295f4640, 0x28, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0931a3900, {0xc0295f4640, 0x29, 0x4b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06c672008, {{0xc0366a9c50, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0579e9700, {{0xc0366a9c50, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ecd9c00, {0x63c85b8, 0xc073d152c0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc073d152c0}, {0x64a6560, 0xc0579e9700}, {0xc0366a9c50, 0x0, 0x30}, {0x174301109ec7b50a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc073d152c0}, {0x6466ed8, 0xc06c672000}, {0xc0366a9c50, 0x28, 0x30}, {0x174301109ec7b50a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc026662b70}, {{0x174301109ec7b50a, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109ec7b50a, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc050f94200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bc951a0a, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0312b0400, {0x63c85b8, 0xc073d152c0}, 0xc050f94200, 0xc0293be500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0312b0400, {0x63c85b8, 0xc073d152c0}, 0xc050f94200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109ec7b50a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109ec7b50a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073d15230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d15230}, {0xc04199b3f8, 0x660089}, 0xc04199b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073d15230}, {0x7fe429309e40}, 0xc050f94000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073d151a0}, 0xc050f94000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}, 0xc0886dc060, 0xc04a6d9d88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}, 0x3, 0xc050a075c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04199b980}}}, {0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}, 0xc0886dc060, 0xc050a075c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073d151a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d151a0}, {0x40, 0xc04199bb70}, 0xc04199bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073d151a0}, {0x4e9d7a0, 0xc050f94000}, 0xc0886dc060, 0xc04a6d9d88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073d151a0}, 0xc04d73b500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0277fe120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0277fe120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802655 [runnable]: internal/poll.runtime_pollWait(0x7fdbaff35fd0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc058618780, 0xc03ef0f800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc058618780, {0xc03ef0f800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc058618780, {0xc03ef0f800, 0x735, 0xc04470a280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03bd0ed30, {0xc03ef0f800, 0x7eda79, 0xc03ef0f805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03b259200, {0xc03ef0f800, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc044e34060, {0xc03ef0f800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04d381078, {0x631b900, 0xc044e34060}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04d380e00, {0x7fe42ab59310, 0xc03b259200}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04d380e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04d380e00, {0xc0271be000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc032b32e40, {0xc0271be000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc040978088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc040978088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc040978660, {0x634aac0, 0xc040978088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc040978000, 0xc02ddbbde0, 0xc08d1d942b, 0xc08d1d942d, {0x635bec0, 0xc00edaa120}, 0xc000d6ae00, 0xc036fa0178, 0xc08d1d9429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc040978000, {0x63c85b8, 0xc077464750}, 0xc02ddbbdd0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc046519038, 0x8}}, 0x0, 0xc0774646f0, 0xc077464720, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc077464750}, {0x643d810, 0xc03b259200}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc025ee8300}, {0x643d810, 0xc03b259200}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc025ee8300}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810601 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea26e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05735a780, 0xc00bc21800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05735a780, {0xc00bc21800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05735a780, {0xc00bc21800, 0x735, 0xc05c757900}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc020fd33b8, {0xc00bc21800, 0xc0, 0xbf}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0ae98a720, {0xc00bc21800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc051b451e8, {0xc00bc21800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0524a93f8, {0x631b900, 0xc051b451e8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0524a9180, {0x7fe42ab59310, 0xc0ae98a720}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0524a9180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0524a9180, {0xc09b49a000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc057eb0d68, {0xc09b49a000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a0512088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a0512088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a0512660, {0x634aac0, 0xc0a0512088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a0512000, 0xc0470d0420, 0xc082d2942b, 0xc082d2942d, {0x635bec0, 0xc063ece5a0}, 0xc000d6ae00, 0xc0b6c2aa68, 0xc082d29429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a0512000, {0x63c85b8, 0xc0266b72c0}, 0xc0470d03d0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc00eaf8009, 0x8}}, 0x0, 0xc0266b7260, 0xc0266b7290, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0266b72c0}, {0x643d810, 0xc0ae98a720}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc051005bc0}, {0x643d810, 0xc0ae98a720}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc051005bc0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804562 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02efd4140, {0x63c85b8, 0xc073c8d350}, 0xc078950bd0, 0x1, 0xc05220d0a8, 0x519bbc, 0xc0615c6720, 0xc05eb59da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc073c8d350}, 0x120, 0xc05220d230, 0x615c66c0, 0x0, 0x660069, 0xc0615c6720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02efd4140, {0x63c85b8, 0xc073c8d350}, 0xc0615c66c0, {{{{0x0, 0x0}, {0xc053c105a0, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073c8d350}, 0x6352410, 0x321a0dd8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ef76e60, {0x63c85b8, 0xc073c8d350}, {0xc0615c65a0, {0x174301103490eb9e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ef76e60, {0x63c85b8, 0xc073c8d350}, 0xc06b233e00, 0xc05220d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073c8d350}, 0x660089, {0xc0615c65a0, {0x174301103490eb9e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02efcb000, {0x63c85b8, 0xc073c8d350}, 0xc0661d5900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103490eb9e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103490eb9e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073c8d2c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073c8d2c0}, {0xc05220f3f8, 0x660089}, 0xc05220f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073c8d2c0}, {0x7fe3ba7bf008}, 0xc0661d5700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073c8d230}, 0xc0661d5700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}, 0xc02b5111a0, 0xc056840138) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}, 0x3, 0xc04f503d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05220f980}}}, {0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}, 0xc02b5111a0, 0xc04f503d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073c8d230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073c8d230}, {0x40, 0xc05220fb70}, 0xc05220fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073c8d230}, {0x4e9d7a0, 0xc0661d5700}, 0xc02b5111a0, 0xc056840138) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073c8d230}, 0xc041a3c4e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0615c6480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0615c6480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811244 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc01df40090}, 0xc024f63100, 0xc083c9f400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01df40090}, 0xc024f63100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110743c5f5b, 0x0, 0x0}, 0xc05f3f9f50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110743c5f5b, 0x0, 0x0}, 0xc05f3f9f50, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021d79f20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021d79f20}, {0xc0293393f8, 0x660089}, 0xc029339400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021d79f20}, {0xc0015221e0}, 0xc024f62f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021d79e60}, 0xc024f62f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}, 0xc040f5c5e0, 0xc049bf05b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}, 0x3, 0xc0286a3f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc029339980}}}, {0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}, 0xc040f5c5e0, 0xc0286a3f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc021d79e60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021d79e60}, {0x40, 0xc029339b70}, 0xc029339ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc021d79e60}, {0x4e9d7a0, 0xc024f62f00}, 0xc040f5c5e0, 0xc049bf05b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc021d79e60}, 0xc026f6b4a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06eb959e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06eb959e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805716 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035a40, {0x63c85b8, 0xc07e2c3680}, 0xc02bcc2180, 0x1, 0xc01f40d0a8, 0x519bbc, 0xc060691380, 0xc05fbe6180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07e2c3680}, 0x120, 0xc01f40d230, 0x60691320, 0xc0414764e0, 0x18, 0xc060691380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035a40, {0x63c85b8, 0xc07e2c3680}, 0xc060691320, {{{{0x0, 0x0}, {0xc04b78e360, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07e2c3680}, 0x6352410, 0x6c0983c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd8b0, {0x63c85b8, 0xc07e2c3680}, {0xc060691200, {0x174301104555bd36, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd8b0, {0x63c85b8, 0xc07e2c3680}, 0xc0675aeaa0, 0xc01f40d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07e2c3680}, 0x660089, {0xc060691200, {0x174301104555bd36, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e096000, {0x63c85b8, 0xc07e2c3680}, 0xc08e372e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104555bd36, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104555bd36, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07e2c35f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e2c35f0}, {0xc01f40f3f8, 0x660089}, 0xc01f40f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07e2c35f0}, {0xc0015221e0}, 0xc08e372b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07e2c3530}, 0xc08e372b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}, 0xc01389b3c0, 0xc02bcc3f38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}, 0x3, 0xc03d087440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f40f980}}}, {0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}, 0xc01389b3c0, 0xc03d087440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07e2c3530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e2c3530}, {0x40, 0xc01f40fb70}, 0xc01f40fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}, 0x20300f, 0x20300f) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07e2c3530}, {0x4e9d7a0, 0xc08e372b00}, 0xc01389b3c0, 0xc02bcc3f38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07e2c3530}, 0xc01f1c2c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0606910e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0606910e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811379 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016d6d2c0, {0x63c85b8, 0xc07489ac90}, 0xc0434fa8b8, 0x1, 0xc0271cf0a8, 0x519bbc, 0xc0d6fe03c0, 0xc01cbf0060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07489ac90}, 0x120, 0xc0271cf230, 0xd6fe0360, 0x0, 0x660069, 0xc0d6fe03c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016d6d2c0, {0x63c85b8, 0xc07489ac90}, 0xc0d6fe0360, {{{{0x0, 0x0}, {0xc051e9c120, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07489ac90}, 0x6352410, 0x7818fda0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce3180, {0x63c85b8, 0xc07489ac90}, {0xc0d6fe0240, {0x1743011076474074, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce3180, {0x63c85b8, 0xc07489ac90}, 0xc01931ee60, 0xc0271cf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07489ac90}, 0x660089, {0xc0d6fe0240, {0x1743011076474074, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016efac00, {0x63c85b8, 0xc07489ac90}, 0xc051991100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07489ab40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07489ab40}, {0xc0271d13f8, 0x660089}, 0xc0271d1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07489ab40}, {0x0}, 0xc051990f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07489aa20}, 0xc051990f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}, 0xc0260ad1a0, 0xc0434fb980) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}, 0x3, 0xc062328580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0271d1980}}}, {0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}, 0xc0260ad1a0, 0xc062328580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07489aa20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07489aa20}, {0x40, 0xc0271d1b70}, 0xc0271d1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07489aa20}, {0x4e9d7a0, 0xc051990f00}, 0xc0260ad1a0, 0xc0434fb980) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07489aa20}, 0xc047638600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06dcb6360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06dcb6360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807362 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0534de300}, 0xc04b20dcb0, {0x646cc48, 0xc0534de3c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0534de300}, 0xedb7a834f, 0xc011fd26c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0534de300}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0606c3560}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc04b20def8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0606c3560}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807628 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc082fa3c20}, 0xc02b4167e0, 0x1, 0xc0078370a8, 0x519bbc, 0xc053ff4600, 0xc0cd8fd5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc082fa3c20}, 0x120, 0xc007837230, 0x53ff45a0, 0x0, 0x660069, 0xc053ff4600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc082fa3c20}, 0xc053ff45a0, {{{{0x0, 0x0}, {0xc03c2fd8c0, 0x24}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc082fa3c20}, 0x6352410, 0x53415a28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc082fa3c20}, {0xc053ff4480, {0x1743011058bf3457, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc082fa3c20}, 0xc0685f15e0, 0xc0078379f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc082fa3c20}, 0x660089, {0xc053ff4480, {0x1743011058bf3457, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc082fa3c20}, 0xc06212e500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058bf3457, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058bf3457, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc082fa3b90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082fa3b90}, {0xc0078393f8, 0x660089}, 0xc007839400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc082fa3b90}, {0x7fe3f7374ed8}, 0xc06212e300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc082fa3b00}, 0xc06212e300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}, 0xc0d55ba080, 0xc03bb18780) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}, 0x3, 0xc0141e5000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007839980}}}, {0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}, 0xc0d55ba080, 0xc0141e5000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc082fa3b00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082fa3b00}, {0x40, 0xc007839b70}, 0xc007839ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc082fa3b00}, {0x4e9d7a0, 0xc06212e300}, 0xc0d55ba080, 0xc03bb18780) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc082fa3b00}, 0xc04e119380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03ff1bc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03ff1bc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812586 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809830 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0837f81b0}, 0xc067b691b8, 0x1, 0xc01aa970a8, 0x519bbc, 0xc0603d9e00, 0xc0352609c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc065940240, {0x63c85b8, 0xc0837f81b0}, 0xc01819f9e0, 0xc01aa971a8, 0x603d9e00, 0x0, 0x0, 0xc0603d9e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0837f81b0}, 0xc0603d9d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0748c46c0, 0x82}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0837f81b0}, 0x6352260, 0x57085918) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0837f81b0}, {0xc0603d9c20, {0x174301102b2a2ba3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0837f81b0}, 0xc072e69f40, 0xc01aa979f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0837f81b0}, 0x660089, {0xc0603d9c20, {0x174301102b2a2ba3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0837f81b0}, 0xc07e10c900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b2a2ba3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b2a2ba3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0837f80f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0837f80f0}, {0xc01aa993f8, 0x660089}, 0xc01aa99400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0837f80f0}, {0xc0015221e0}, 0xc07e10c700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0837f8060}, 0xc07e10c700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}, 0xc012c80420, 0xc047936c60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}, 0x3, 0xc0c4929b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aa99980}}}, {0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}, 0xc012c80420, 0xc0c4929b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0837f8060}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0837f8060}, {0x40, 0xc01aa99b70}, 0xc01aa99ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0837f8060}, {0x4e9d7a0, 0xc07e10c700}, 0xc012c80420, 0xc047936c60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0837f8060}, 0xc029d3f7a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0603d9b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0603d9b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811355 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc016c4d000, {0x63c85b8, 0xc060dcf320}, 0xc044a32200, 0xc0c2a71a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc060dcf320}, 0xc044a32200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060dcf290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060dcf290}, {0xc00ef953f8, 0x660089}, 0xc00ef95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060dcf290}, {0x7fe137a44f60}, 0xc055383f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc060dcf200}, 0xc055383f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}, 0xc037ae3f40, 0xc01dfebef0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}, 0x3, 0xc02a6263c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ef95980}}}, {0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}, 0xc037ae3f40, 0xc02a6263c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc060dcf200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060dcf200}, {0x40, 0xc00ef95b70}, 0xc00ef95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc060dcf200}, {0x4e9d7a0, 0xc055383f00}, 0xc037ae3f40, 0xc01dfebef0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc060dcf200}, 0xc04b8a4ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc01cbf0480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc01cbf0480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219130556 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe796d8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc089aad680, 0xc014a15000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc089aad680, {0xc014a15000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc089aad680, {0xc014a15000, 0x0, 0xc0582bcbc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01e6516e0, {0xc014a15000, 0x100, 0x6}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02cd78fc0, {0xc014a15000, 0x7, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0408ceed0, {0xc014a15000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc057c285f8, {0x631b900, 0xc0408ceed0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc057c28380, {0x7fe42ab59310, 0xc02cd78fc0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc057c28380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc057c28380, {0xc050509000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc035a4c750, {0xc050509000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc087bbeb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc087bbeb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc087bbf0e0, {0x634aac0, 0xc087bbeb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc087bbea80, 0xc0628e9800, 0xc0582bd42b, 0xc0582bd42d, {0x635bec0, 0xc0107e97a0}, 0xc000d6ae00, 0xc0505aa648, 0xc0582bd429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc087bbea80, {0x63c85b8, 0xc06c303bf0}, 0xc0628e97f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0999f3025, 0x8}}, 0x0, 0xc06c303b90, 0xc06c303bc0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06c303bf0}, {0x643d810, 0xc02cd78fc0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc021c59440}, {0x643d810, 0xc02cd78fc0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc021c59440}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805747 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc011ca7400, {0x63c85b8, 0xc04261cff0}, 0xc07926f400, 0xc081ccae60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc011ca7400, {0x63c85b8, 0xc04261cff0}, 0xc07926f400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011047f75a42, 0x0, 0x0}, 0xc05ca1e9e0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011047f75a42, 0x0, 0x0}, 0xc05ca1e9e0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04261cf60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04261cf60}, {0xc00384d3f8, 0x660089}, 0xc00384d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04261cf60}, {0x7fe3fb715400}, 0xc07926f100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04261ced0}, 0xc07926f100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}, 0xc03f52f360, 0xc03c0496c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}, 0x3, 0xc016845fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00384d980}}}, {0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}, 0xc03f52f360, 0xc016845fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04261ced0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04261ced0}, {0x40, 0xc00384db70}, 0xc00384dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04261ced0}, {0x4e9d7a0, 0xc07926f100}, 0xc03f52f360, 0xc03c0496c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04261ced0}, 0xc04e85b020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093732000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093732000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802857 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01896a280, {0x63c85b8, 0xc066b98510}, 0xc02765f848, 0x1, 0x0, 0x1, 0xc075c2b5c0, 0xc049fa8600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc066b98510}, 0x120, 0xc05edb2728, 0x75c2b560, 0x0, 0x660069, 0xc075c2b5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01896a280, {0x63c85b8, 0xc066b98510}, 0xc075c2b560, {{{{0x0, 0x0}, {0xc0079046c0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc066b98510}, 0x6352410, 0x561afe80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01893e3c0, {0x63c85b8, 0xc066b98510}, {0xc0678f39e0, {0x1743011025b3ca1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01893e3c0, {0x63c85b8, 0xc066b98510}, 0xc052ea7ea0, 0xc07b0d4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066b98510}, 0x660089, {0xc0678f39e0, {0x1743011025b3ca1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01898a000, {0x63c85b8, 0xc066b98510}, 0xc0781f2e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025b3ca1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025b3ca1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066b98480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066b98480}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066b98480}, {0xc00102e4e0}, 0xc0781f2d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066b98420}, 0xc0781f2d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0476bbf20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812059 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc07b5bac30}, 0xc013d6dbc0, 0x1, 0xc036d4b0a8, 0x519bbc, 0xc02fc94420, 0xc05a937860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05b281d40, {0x63c85b8, 0xc07b5bac30}, 0xc05b281e30, 0xc036d4b1a8, 0x26, 0x40, 0xffffffffffffffff, 0xc02fc94420, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc07b5bac30}, 0xc02fc94360, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc08c670900, 0x45}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07b5bac30}, 0x6352260, 0x33a924c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc07b5bac30}, {0xc02fc94240, {0x1743011041c6d918, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc07b5bac30}, 0xc0413c7860, 0xc036d4b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07b5bac30}, 0x660089, {0xc02fc94240, {0x1743011041c6d918, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc07b5bac30}, 0xc06f31d900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041c6d918, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041c6d918, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07b5baba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b5baba0}, {0xc036d4d3f8, 0x660089}, 0xc036d4d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07b5baba0}, {0x7fe422bd6838}, 0xc06f31d700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07b5baae0}, 0xc06f31d700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}, 0xc0262c70c0, 0xc03c538510) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}, 0x3, 0xc07810c6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc036d4d980}}}, {0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}, 0xc0262c70c0, 0xc07810c6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07b5baae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b5baae0}, {0x40, 0xc036d4db70}, 0xc036d4dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07b5baae0}, {0x4e9d7a0, 0xc06f31d700}, 0xc0262c70c0, 0xc03c538510) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07b5baae0}, 0xc0408ea4e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00d353c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00d353c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812669 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800448 [semacquire]: sync.runtime_SemacquireMutex(0x58e3045, 0x40, 0x7fe3ea2e29a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc063390d00, 0x300000002, 0xc063390d00) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc000094800, 0xc083e1a420, 0x4571d1, 0x424a070) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7236e27, 0x3b81}, 0x0, 0xc03694cff8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc03694cd80, {0xffffffffffffff15, 0xc083e1a5b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc03694cd80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc03694cd80, {0xc0e7385d00, 0x32, 0x77}, 0x13, 0x15) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc03694cd80, {0xc01e5a2080, 0x32, 0x77}, {0xc0e7385d00, 0x51c9860, 0xc0e7385d00}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc03694cd80, {0xc01e5a2080, 0x32, 0x80}, {0xc0e7385d00, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d461e98, {0xc01e5a2080, 0x32, 0x80}, {0xc0e7385d00, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d4607c0, {0xc0e7385d00, 0x77, 0x80}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d4607c0, {0xc01e5a2080, 0x77, 0xc0e7385d00}, {0xc0e7385d00, 0xc077eb7320, 0x31}, 0x8c) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d460500, {0xc0e7385d00, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc088f743a8, {{0xc077eb7320, 0x31, 0x8c}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0638d6a00, {{0xc054f744b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc06055dc00, {0x63c85b8, 0xc05459df20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc05459df20}, {0x64a6560, 0xc0638d6a00}, {0xc054f744b0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc05459df20}, {0x6466ed8, 0xc088f74100}, {0xc054f744b0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07c51b398}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc00d19da00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b0e000, {0x63c85b8, 0xc05459df20}, 0xc00d19da00, 0xc077d0b9a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b0e000, {0x63c85b8, 0xc05459df20}, 0xc00d19da00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05459de90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05459de90}, {0xc0933e48f0, 0x660089}, 0xc0933e48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05459de90}, {0xc00102e4e0}, 0xc00d19d600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05459de30}, 0xc00d19d600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06b4b8b70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 206718584 [sync.Cond.Wait, 1 minutes]: sync.runtime_notifyListWait(0xc0681590d0, 0x621) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc03fdb2000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc03fdb2660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003258c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003258c00, {0x63c8510, 0xc068159100}, 0xc045871dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc00f123910) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc068159100}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc00f123910) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219798750 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea3220, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc063dae900, 0xc01d96a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc063dae900, {0xc01d96a000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc063dae900, {0xc01d96a000, 0x735, 0xc05453a500}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0355e67a8, {0xc01d96a000, 0x7eda79, 0xc01d96a005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0892d7ce0, {0xc01d96a000, 0xc0680a6c01, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2691620, {0xc01d96a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06ae23778, {0x631b900, 0xc0b2691620}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06ae23500, {0x7fe42ab59310, 0xc0892d7ce0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06ae23500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06ae23500, {0xc0342d8000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc026959bf0, {0xc0342d8000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc052310b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc052310b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0523110e0, {0x634aac0, 0xc052310b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc052310a80, 0xc05b056ce0, 0xc0680a742b, 0xc0680a742d, {0x635bec0, 0xc09b2e5950}, 0xc000d6ae00, 0xc0043ac2e8, 0xc0680a7429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc052310a80, {0x63c85b8, 0xc0493fd9b0}, 0xc05b056cd0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc04f75b021, 0x8}}, 0x0, 0xc0493fd950, 0xc0493fd980, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0493fd9b0}, {0x643d810, 0xc0892d7ce0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc064913590}, {0x643d810, 0xc0892d7ce0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc064913590}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810185 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0214da000, {0x63c85b8, 0xc02d7b4bd0}, 0xc032cf7b00, 0xc04f60c000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc02d7b4bd0}, 0xc032cf7b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bef38a9, 0x0, 0x0}, 0xc03a65c0a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bef38a9, 0x0, 0x0}, 0xc03a65c0a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02d7b4b40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d7b4b40}, {0xc0234ab3f8, 0x660089}, 0xc0234ab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02d7b4b40}, {0x0}, 0xc032cf7900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02d7b4ab0}, 0xc032cf7900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}, 0xc07cd09fc0, 0xc02532cd38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}, 0x3, 0xc0887607c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0234ab980}}}, {0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}, 0xc07cd09fc0, 0xc0887607c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02d7b4ab0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d7b4ab0}, {0x40, 0xc0234abb70}, 0xc0234abab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02d7b4ab0}, {0x4e9d7a0, 0xc032cf7900}, 0xc07cd09fc0, 0xc02532cd38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02d7b4ab0}, 0xc03ee832c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060a926c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060a926c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812682 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).GetLiveness(0xc007b682c0, 0x1f) github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:1110 +0x2fa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).leaseStatus(_, {_, _}, {{0x1742dae73a3fe053, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:657 +0x23f github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).leaseStatusForRequestRLocked(0x0, {0x63c85b8, 0xc05959f7a0}, {0x968709dfca154ea3, 0x460a637b, 0x5}, {0xc04ca9e270, _, _}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:738 +0x19e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).checkGCThresholdAndLeaseRLocked(0xc023992c00, {0x63c85b8, 0xc05959f7a0}, 0xc0711f1b00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:1460 +0x159 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).checkExecutionCanProceed(0xc023992c00, {0x63c85b8, 0xc05959f7a0}, 0xc0711f1b00, 0xc020d53680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:1408 +0x27e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc023992c00, {0x63c85b8, 0xc05959f7a0}, 0xc0711f1b00, 0xc020d53680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:42 +0x13b github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023992c00, {0x63c85b8, 0xc05959f7a0}, 0xc0711f1b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108924fa74, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108924fa74, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05959f710}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05959f710}, {0xc043fad3f8, 0x660089}, 0xc043fad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05959f710}, {0x5398e984f1ba779c}, 0xc0711f1900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05959f680}, 0xc0711f1900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}, 0xc054b1a700, 0xc017daff68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}, 0x3, 0xc050a161c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043fad980}}}, {0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}, 0xc054b1a700, 0xc050a161c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05959f680}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05959f680}, {0x40, 0xc043fadb70}, 0xc043fadab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05959f680}, {0x4e9d7a0, 0xc0711f1900}, 0xc054b1a700, 0xc017daff68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05959f680}, 0xc0958ee840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03d0ccea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03d0ccea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808081 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc045351530}, 0xc02d195428, 0x1, 0x0, 0x1, 0xc0a8d20ba0, 0xc061fcd8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc045351530}, 0x1e0, 0xc004fa4728, 0xa8d20b40, 0x0, 0x660069, 0xc0a8d20ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc045351530}, 0xc0a8d20b40, {{{{0x0, 0x0}, {0xc0566a86c0, 0x48}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc045351530}, 0x6352410, 0x13b806c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc045351530}, {0xc07c17a7e0, {0x174301105b1ef448, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc045351530}, 0xc06384d360, 0xc076022ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc045351530}, 0x660089, {0xc07c17a7e0, {0x174301105b1ef448, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc045351530}, 0xc0b3b9cd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b1ef448, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b1ef448, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0453514a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0453514a0}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0453514a0}, {0xc00102e4e0}, 0xc0b3b9cc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045351440}, 0xc0b3b9cc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0782fa030}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 218871406 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4db38, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc053625c80, 0xc0a938f800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc053625c80, {0xc0a938f800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc053625c80, {0xc0a938f800, 0x735, 0xc054835400}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01b1221d8, {0xc0a938f800, 0x40, 0x3f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc031223e00, {0xc0a938f800, 0xc078312c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02f3dd368, {0xc0a938f800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc062cc3af8, {0x631b900, 0xc02f3dd368}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc062cc3880, {0x7fe42ab59310, 0xc031223e00}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc062cc3880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc062cc3880, {0xc0b1778000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03ac1dda0, {0xc0b1778000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc074caf588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc074caf588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc074cafb60, {0x634aac0, 0xc074caf588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc074caf500, 0xc04fefecf0, 0xc06011142b, 0xc06011142d, {0x635bec0, 0xc06425cab0}, 0xc000d6ae00, 0xc03b347da8, 0xc060111429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc074caf500, {0x63c85b8, 0xc066058ff0}, 0xc04fefece0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0b174c03b, 0x8}}, 0x0, 0xc066058f90, 0xc066058fc0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc066058ff0}, {0x643d810, 0xc031223e00}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0462b6ea0}, {0x643d810, 0xc031223e00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0462b6ea0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219694612 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021c87400, {0x63c85b8, 0xc08a8c8ab0}, 0xc08b761300, 0xc06877cd20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c87400, {0x63c85b8, 0xc08a8c8ab0}, 0xc08b761300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e6522ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e6522ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08a8c8a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a8c8a20}, {0xc0932b47f8, 0x660089}, 0xc0932b4800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08a8c8a20}, {0xc00102e4e0}, 0xc08b761200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08a8c89c0}, 0xc08b761200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc028603c30, {0x63c85b8, 0xc08a8c8870}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc041968600, {0x63c85b8, 0xc08a8c8870}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc041968600, {0x63c8510, 0xc089ba6800}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc041968600, {0x63c8510, 0xc089ba6800}, {0x63f7db8, 0x9bf6f48}, 0xc0932b9088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc041968600, {0x63c85b8, 0xc076047a40}, {{0x63f7db8, 0x9bf6f48}, {0xc07718177d, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc041968600, {0x63c85b8, 0xc076047a40}, {{0x63f7db8, 0x9bf6f48}, {0xc07718177d, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc07718177d, 0x6}, 0x0, 0x0}, {0x34343e83, 0xedb7a834f, 0x0}, {0x34344823, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc041968600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc041968600, {0x63c8510, 0xc053899c00}, 0xc058bd3dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08c064bf0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc053899c00}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08c064bf0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219711543 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc09ad00c60, {0x63c85b8, 0xc06c2f8480}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc07393c000, {0x63c85b8, 0xc06c2f8480}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc07393c000, {0x63c8510, 0xc03ee23680}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc07393c000, {0x63c8510, 0xc03ee23680}, 0xc084d6e9c0, {0x7fe42b790cf0, 0xc0aa9b7bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc07393c000, {0x63c85b8, 0xc046ae89f0}, {{0x63f88f8, 0xc084d6e9c0}, {0xc0b1723d51, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc07393c000, {0x63c85b8, 0xc046ae89f0}, {{0x63f88f8, 0xc084d6e9c0}, {0xc0b1723d51, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc084d6e9c0}, {0xc0b1723d51, 0x23}, 0x0, 0x0}, {0x2374ae41, 0xedb7a834f, 0x0}, {0x2374b4d1, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc07393c000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc07393c000, {0x63c8510, 0xc0712d5980}, 0xc07c39ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06c818c20) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0712d5980}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06c818c20) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804728 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc02784b950}, 0xc07820b860, 0x1, 0x0, 0x1, 0xc0545ed320, 0xc04a7849c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02784b950}, 0x1e0, 0xc07b632728, 0x545ed2c0, 0xc01e2df040, 0x18, 0xc0545ed320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc02784b950}, 0xc0545ed2c0, {{{{0x0, 0x0}, {0xc07c2d3b00, 0x54}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc02784b950}, 0xc002619270, 0x8da96920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc02784b950}, {0xc06d3b65a0, {0x17430110310ddab1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02784b950}, 0xc05002ae60, 0xc08da96ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02784b950}, 0x660089, {0xc06d3b65a0, {0x17430110310ddab1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02784b950}, 0xc0bc3cb200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02784b8c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02784b8c0}, {0xc0936928f0, 0x660089}, 0xc0936928f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02784b8c0}, {0xc00102e4e0}, 0xc0bc3cb100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02784b860}, 0xc0bc3cb100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc073b6d860}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219727460 [runnable]: internal/poll.runtime_pollWait(0x7fdbaffb3380, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc090e11680, 0xc06d688000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc090e11680, {0xc06d688000, 0x359d, 0x359d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc090e11680, {0xc06d688000, 0x359d, 0xc059945cc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc046cedbb8, {0xc06d688000, 0x7eda79, 0xc06d688005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc023077b60, {0xc06d688000, 0xc07b9eec60, 0x359d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc044e346a8, {0xc06d688000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04b8b25f8, {0x631b900, 0xc044e346a8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04b8b2380, {0x7fe42ab59310, 0xc023077b60}, 0x359d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04b8b2380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04b8b2380, {0xc00d4f1000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc052e48120, {0xc00d4f1000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03bf64b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03bf64b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03bf650e0, {0x634aac0, 0xc03bf64b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03bf64a80, 0xc07c0f9130, 0xc07b9ef42b, 0xc07b9ef42d, {0x635bec0, 0xc0186619e0}, 0xc000d6ae00, 0xc0573c79e8, 0xc07b9ef429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03bf64a80, {0x63c85b8, 0xc01d502ff0}, 0xc07c0f9120, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a28a8037, 0x8}}, 0x0, 0xc01d502f90, 0xc01d502fc0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc01d502ff0}, {0x643d810, 0xc023077b60}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0755cfaa0}, {0x643d810, 0xc023077b60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0755cfaa0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802562 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc084296548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0842965b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0920ad6f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc03fc57880, {0xc01cdadce0, 0x2b, 0x2b}, {0xc01cdad4d0, 0x2b, 0x2b}, 0x1, 0xd6) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0920ad588, {0xc01cdadce0, 0x1, 0x2b}, {0xc01cdad4d0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0920ad588, {0xc01cdadce0, 0x2b, 0x2b}, {0xc01cdad4d0, 0x2b, 0x2b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0920accc0, {0xc01cdad4d0, 0x2b, 0x2b}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0920accc0, {0xc01cdadce0, 0x2b, 0xc01cdad4d0}, {0xc01cdad4d0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0920aca00, {0xc01cdad4d0, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc09cf8ec08, {{0xc01956bc80, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0988e3800, {{0xc01956bc80, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc047980c00, {0x63c85b8, 0xc045ea45a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc045ea45a0}, {0x64a6560, 0xc0988e3800}, {0xc01956bc80, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc045ea45a0}, {0x6466ed8, 0xc09cf8ec00}, {0xc01956bc80, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0a7f44090}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc07acea800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f1c000, {0x63c85b8, 0xc045ea45a0}, 0xc07acea800, 0xc074471400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f1c000, {0x63c85b8, 0xc045ea45a0}, 0xc07acea800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045ea4510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045ea4510}, {0xc0d68328f0, 0x660089}, 0xc0d68328f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045ea4510}, {0x24}, 0xc07acea700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045ea44b0}, 0xc07acea700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803478 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0190e3e60}, 0xc0434fadc8, 0x1, 0xc06e5c50a8, 0x519bbc, 0xc047358420, 0xc03e3cbe60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0190e3e60}, 0x1e0, 0xc06e5c5230, 0x473583c0, 0x0, 0x660069, 0xc047358420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0190e3e60}, 0xc0473583c0, {{{{0x0, 0x0}, {0xc051d3f8c0, 0x40}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0190e3e60}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0190e3e60}, {0xc03b85e000, {0x174301102c9a3fc8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0190e3e60}, 0xc0291925a0, 0xc06e5c59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0190e3e60}, 0x660089, {0xc03b85e000, {0x174301102c9a3fc8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0190e3e60}, 0xc00fe99500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102c9a3fc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102c9a3fc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0190e3ce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0190e3ce0}, {0xc06e5c73f8, 0x660089}, 0xc06e5c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0190e3ce0}, {0xc01c2df7e8}, 0xc00fe99300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0190e3b60}, 0xc00fe99300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}, 0xc04622f3a0, 0xc014ba8840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}, 0x3, 0xc043752480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06e5c7980}}}, {0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}, 0xc04622f3a0, 0xc043752480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0190e3b60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0190e3b60}, {0x40, 0xc06e5c7b70}, 0xc06e5c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0190e3b60}, {0x4e9d7a0, 0xc00fe99300}, 0xc04622f3a0, 0xc014ba8840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0190e3b60}, 0xc042573920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc039503e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc039503e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812741 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219143256 [select, 1 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0a43ddd80) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 219694643 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea34f0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06f220a00, 0xc053aec000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06f220a00, {0xc053aec000, 0x2b92, 0x2b92}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06f220a00, {0xc053aec000, 0x2b92, 0xc0458ac3c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc04207e200, {0xc053aec000, 0x7eda79, 0xc053aec005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04b46bc80, {0xc053aec000, 0xc01f2a0c60, 0x2b92}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02f753b78, {0xc053aec000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc021dcc5f8, {0x631b900, 0xc02f753b78}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc021dcc380, {0x7fe42ab59310, 0xc04b46bc80}, 0x2b92) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc021dcc380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc021dcc380, {0xc097a84000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc09e286f48, {0xc097a84000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0986d4b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0986d4b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0986d50e0, {0x634aac0, 0xc0986d4b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0986d4a80, 0xc08c064bd0, 0xc01f2a142b, 0xc01f2a142d, {0x635bec0, 0xc01184b8c0}, 0xc000d6ae00, 0xc060124e58, 0xc01f2a1429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0986d4a80, {0x63c85b8, 0xc0bb861320}, 0xc08c064bc0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc035557021, 0x8}}, 0x0, 0xc0bb8612c0, 0xc0bb8612f0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0bb861320}, {0x643d810, 0xc04b46bc80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0ac519c50}, {0x643d810, 0xc04b46bc80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0ac519c50}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809902 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc051ac6750, {0x63c85b8, 0xc01de4e7e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc01de4e7e0}, {0x4f67ad2, 0x6446ff8}, 0xc027908fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219802576 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc099a44548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc099a445b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07d5f80f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc059e62a80, {0xc0318b4900, 0x2b, 0x80}, {0xc01f988fc0, 0x2b, 0x6e}, 0x1, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07d5f7f88, {0xc0318b4900, 0x1, 0x6e}, {0xc01f988fc0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d5f7f88, {0xc0318b4900, 0x2b, 0x80}, {0xc01f988fc0, 0x2b, 0x6e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d5f76c0, {0xc01f988fc0, 0x6e, 0x80}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d5f76c0, {0xc0318b4900, 0x2b, 0xc01f988fc0}, {0xc01f988fc0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d5f7400, {0xc01f988fc0, 0x2b, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc051349708, {{0xc0cae85b00, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0629fa300, {{0xc0cae85b00, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ff99800, {0x63c85b8, 0xc02de81b00}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02de81b00}, {0x64a6560, 0xc0629fa300}, {0xc0cae85b00, 0xc9c84a5e6380c2b, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02de81b00}, {0x6466ed8, 0xc051349700}, {0xc0cae85b00, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc079319ea8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06b9a2c00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4a000, {0x63c85b8, 0xc02de81b00}, 0xc06b9a2c00, 0xc050935040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4a000, {0x63c85b8, 0xc02de81b00}, 0xc06b9a2c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02de81920}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02de81920}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02de81920}, {0xc00102e4e0}, 0xc06b9a2b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02de815c0}, 0xc06b9a2b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220bd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805102 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024a0de00, {0x63c85b8, 0xc0778f8030}, 0xc0205de8b8, 0x1, 0x0, 0x1, 0xc072af6060, 0xc035260de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0778f8030}, 0x120, 0xc056e50728, 0x72af6000, 0xc04966b1e0, 0x18, 0xc072af6060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024a0de00, {0x63c85b8, 0xc0778f8030}, 0xc072af6000, {{{{0x0, 0x0}, {0xc04ebe5d40, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0778f8030}, 0x6352410, 0x71c3e978) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0249476d0, {0x63c85b8, 0xc0778f8030}, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0249476d0, {0x63c85b8, 0xc0778f8030}, 0xc054757d60, 0xc0829c0ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0778f8030}, 0x660089, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024a87000, {0x63c85b8, 0xc0778f8030}, 0xc04d219f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053cc3f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053cc3f80}, {0xc0936988f0, 0x660089}, 0xc0936988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053cc3f80}, {0xc00102e4e0}, 0xc04d219d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053cc3f20}, 0xc04d219d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fcdbda0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808533 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc0435426c0}, 0xc02c3ce060, 0x1, 0xc00696b0a8, 0x519bbc, 0xc05863fc80, 0xc092ce9c60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0435426c0}, 0x120, 0xc00696b230, 0x5863fc20, 0x0, 0x660069, 0xc05863fc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc0435426c0}, 0xc05863fc20, {{{{0x0, 0x0}, {0xc06540e6c0, 0x10}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0435426c0}, 0x63526b0, 0x1b929cf0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc0435426c0}, {0xc05863fb00, {0x174301105cfd6a7d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc0435426c0}, 0xc079818e60, 0xc00696b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0435426c0}, 0x660089, {0xc05863fb00, {0x174301105cfd6a7d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc0435426c0}, 0xc071cfd000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105cfd6a7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105cfd6a7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0435425a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0435425a0}, {0xc00696d3f8, 0x660089}, 0xc00696d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0435425a0}, {0x7fe425726468}, 0xc071cfce00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0435424e0}, 0xc071cfce00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}, 0xc03a377d00, 0xc035a18fd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}, 0x3, 0xc07a553a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00696d980}}}, {0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}, 0xc03a377d00, 0xc07a553a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0435424e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0435424e0}, {0x40, 0xc00696db70}, 0xc00696dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0435424e0}, {0x4e9d7a0, 0xc071cfce00}, 0xc03a377d00, 0xc035a18fd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0435424e0}, 0xc02e28c6c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05863f8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05863f8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806963 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ad17cc0, {0x63c85b8, 0xc0938b0e10}, 0xc02d6d0ff0, 0x1, 0xc004e930a8, 0x519bbc, 0xc0d3ca6060, 0xc05c0b2180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0938b0e10}, 0x120, 0xc004e93230, 0xd3ca6000, 0x0, 0x660069, 0xc0d3ca6060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ad17cc0, {0x63c85b8, 0xc0938b0e10}, 0xc0d3ca6000, {{{{0x0, 0x0}, {0xc07abd2360, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0938b0e10}, 0x6352410, 0x25ab65b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ad465f0, {0x63c85b8, 0xc0938b0e10}, {0xc000b4be60, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ad465f0, {0x63c85b8, 0xc0938b0e10}, 0xc0761d06e0, 0xc004e939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0938b0e10}, 0x660089, {0xc000b4be60, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ad76000, {0x63c85b8, 0xc0938b0e10}, 0xc078ff8e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0938b0d80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0938b0d80}, {0xc004e953f8, 0x660089}, 0xc004e95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0938b0d80}, {0x0}, 0xc078ff8c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0938b0cf0}, 0xc078ff8c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}, 0xc08e4ebd60, 0xc02e15de30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}, 0x3, 0xc08f432bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004e95980}}}, {0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}, 0xc08e4ebd60, 0xc08f432bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0938b0cf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0938b0cf0}, {0x40, 0xc004e95b70}, 0xc004e95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0938b0cf0}, {0x4e9d7a0, 0xc078ff8c00}, 0xc08e4ebd60, 0xc02e15de30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0938b0cf0}, 0xc067e10540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc000b4bd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc000b4bd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811611 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc03e596420}, 0xc000f13d40, 0x1, 0x70801818508804, 0x37e11d600, 0xc063c9b140, 0xc04e7d84e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03e596420}, 0x120, 0xc035be9230, 0x63c9b0e0, 0x0, 0x660069, 0xc063c9b140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc03e596420}, 0xc063c9b0e0, {{{{0x0, 0x0}, {0xc021b7e240, 0x11}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03e596420}, 0x6352410, 0x2c65fe98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc03e596420}, {0xc063c9afc0, {0x174301107b1b7318, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc03e596420}, 0xc052ea61e0, 0xc035be99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03e596420}, 0x660089, {0xc063c9afc0, {0x174301107b1b7318, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc03e596420}, 0xc07afa8b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b1b7318, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b1b7318, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03e596390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03e596390}, {0xc035beb3f8, 0x660089}, 0xc035beb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03e596390}, {0x7fe394454dd8}, 0xc07afa8800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03e596300}, 0xc07afa8800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}, 0xc044a36cc0, 0xc000f13d10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}, 0x3, 0xc063201f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc035beb980}}}, {0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}, 0xc044a36cc0, 0xc063201f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03e596300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03e596300}, {0x40, 0xc035bebb70}, 0xc035bebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03e596300}, {0x4e9d7a0, 0xc07afa8800}, 0xc044a36cc0, 0xc000f13d10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03e596300}, 0xc020b99800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc063c9aea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc063c9aea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809905 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ecbf400, {0x63c85b8, 0xc076ed9770}, 0xc020c859e0, 0x1, 0xc0054f70a8, 0x519bbc, 0xc07aafa600, 0xc07c3bede0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc076ed9770}, 0x120, 0xc0054f7230, 0x7aafa5a0, 0x0, 0x660069, 0xc07aafa600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ecbf400, {0x63c85b8, 0xc076ed9770}, 0xc07aafa5a0, {{{{0x0, 0x0}, {0xc024d105a0, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc076ed9770}, 0x6352410, 0x43a28750) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01eb85e50, {0x63c85b8, 0xc076ed9770}, {0xc07aafa480, {0x1743011065f7b8c6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01eb85e50, {0x63c85b8, 0xc076ed9770}, 0xc067b10dc0, 0xc0054f79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc076ed9770}, 0x660089, {0xc07aafa480, {0x1743011065f7b8c6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ed2e800, {0x63c85b8, 0xc076ed9770}, 0xc08b934c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065f7b8c6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065f7b8c6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076ed96e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076ed96e0}, {0xc0054f93f8, 0x660089}, 0xc0054f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076ed96e0}, {0x40df00bf96283369}, 0xc08b934a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076ed9650}, 0xc08b934a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}, 0xc01e9ee240, 0xc02febed20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}, 0x3, 0xc0412e5780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0054f9980}}}, {0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}, 0xc01e9ee240, 0xc0412e5780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076ed9650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076ed9650}, {0x40, 0xc0054f9b70}, 0xc0054f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076ed9650}, {0x4e9d7a0, 0xc08b934a00}, 0xc01e9ee240, 0xc02febed20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076ed9650}, 0xc052799ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07aafa360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07aafa360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802661 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0218b1000, {0x63c85b8, 0xc06ff877a0}, 0xc050d53000, 0xc07445fd60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b1000, {0x63c85b8, 0xc06ff877a0}, 0xc050d53000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011000874ea8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011000874ea8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f116090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f116090}, {0xc017ecf3f8, 0x660089}, 0xc017ecf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f116090}, {0x33}, 0xc0890a8900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050e35a70}, 0xc0890a8900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}, 0xc039a9fa40, 0xc07db20960) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}, 0x3, 0xc05f65e6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017ecf980}}}, {0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}, 0xc039a9fa40, 0xc05f65e6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc050e35a70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050e35a70}, {0x40, 0xc017ecfb70}, 0xc017ecfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc050e35a70}, {0x4e9d7a0, 0xc0890a8900}, 0xc039a9fa40, 0xc07db20960) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc050e35a70}, 0xc061977320, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04b03a900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04b03a900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809844 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00d7f9680, {0x63c85b8, 0xc045fd8a50}, 0xc047936df8, 0x1, 0xc01fcf90a8, 0x519bbc, 0xc085e0a4e0, 0xc08374f5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc045fd8a50}, 0x120, 0xc01fcf9230, 0x85e0a480, 0x0, 0x660069, 0xc085e0a4e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00d7f9680, {0x63c85b8, 0xc045fd8a50}, 0xc085e0a480, {{{{0x0, 0x0}, {0xc0d08af320, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc045fd8a50}, 0x6352410, 0x21726d58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc008446190, {0x63c85b8, 0xc045fd8a50}, {0xc085e0a360, {0x174301105ee9eb70, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc008446190, {0x63c85b8, 0xc045fd8a50}, 0xc03b364c80, 0xc01fcf99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc045fd8a50}, 0x660089, {0xc085e0a360, {0x174301105ee9eb70, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000de5000, {0x63c85b8, 0xc045fd8a50}, 0xc047d12300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105ee9eb70, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105ee9eb70, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045fd89c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045fd89c0}, {0xc01fcfb3f8, 0x660089}, 0xc01fcfb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045fd89c0}, {0xc03cc8f540}, 0xc047d12100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045fd8930}, 0xc047d12100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}, 0xc01f170660, 0xc03924a3a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}, 0x3, 0xc030295680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fcfb980}}}, {0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}, 0xc01f170660, 0xc030295680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045fd8930}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045fd8930}, {0x40, 0xc01fcfbb70}, 0xc01fcfbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045fd8930}, {0x4e9d7a0, 0xc047d12100}, 0xc01f170660, 0xc03924a3a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045fd8930}, 0xc029b6b9e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0744d98c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0744d98c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219369756 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafee9a60, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0c5ce6380, 0xc072c71000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0c5ce6380, {0xc072c71000, 0x2b97, 0x2b97}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0c5ce6380, {0xc072c71000, 0x2b97, 0xc0646828c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc02d7b6290, {0xc072c71000, 0x7eda79, 0xc072c71005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01ebabb60, {0xc072c71000, 0x2, 0x2b97}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc003ce3e18, {0xc072c71000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0749cc978, {0x631b900, 0xc003ce3e18}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0749cc700, {0x7fe42ab59310, 0xc01ebabb60}, 0x2b97) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0749cc700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0749cc700, {0xc096108000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02fde78f0, {0xc096108000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a0f90088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a0f90088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a0f90660, {0x634aac0, 0xc0a0f90088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a0f90000, 0xc03e54df80, 0xc056ab542b, 0xc056ab542d, {0x635bec0, 0xc0486be6c0}, 0xc000d6ae00, 0xc07551c3c8, 0xc056ab5429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a0f90000, {0x63c85b8, 0xc019ca5530}, 0xc03e54df70, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0749fe037, 0x8}}, 0x0, 0xc019ca54d0, 0xc019ca5500, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc019ca5530}, {0x643d810, 0xc01ebabb60}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc076f97620}, {0x643d810, 0xc01ebabb60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc076f97620}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809780 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc011ca7400, {0x63c85b8, 0xc043aa78c0}, 0xc057e6b300, 0xc065964280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc011ca7400, {0x63c85b8, 0xc043aa78c0}, 0xc057e6b300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066f46fa1, 0x0, 0x0}, 0xc01e914620, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066f46fa1, 0x0, 0x0}, 0xc01e914620, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043aa67e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043aa67e0}, {0xc006c1f3f8, 0x660089}, 0xc006c1f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043aa67e0}, {0x3}, 0xc057e6b000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043aa6540}, 0xc057e6b000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}, 0xc04cf67dc0, 0xc0403c1968) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}, 0x3, 0xc046a14f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006c1f980}}}, {0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}, 0xc04cf67dc0, 0xc046a14f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc043aa6540}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043aa6540}, {0x40, 0xc006c1fb70}, 0xc006c1fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc043aa6540}, {0x4e9d7a0, 0xc057e6b000}, 0xc04cf67dc0, 0xc0403c1968) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc043aa6540}, 0xc06c740480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07d459440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07d459440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812404 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0613ef7d0, {0x63c85b8, 0xc06be67350}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc06be67350}, {0x4f67ad2, 0x6446ff8}, 0xc07e11cfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219794239 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc03a08dce0, {0x63c85b8, 0xc044fe6450}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc09c1cf300, {0x63c85b8, 0xc044fe6450}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc09c1cf300, {0x63c8510, 0xc0affdaf40}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc09c1cf300, {0x63c8510, 0xc0affdaf40}, 0xc0868062a0, {0x7fe42b790cf0, 0xc04400b140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc09c1cf300, {0x63c85b8, 0xc01881bf20}, {{0x63f88f8, 0xc0868062a0}, {0xc078780789, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc09c1cf300, {0x63c85b8, 0xc01881bf20}, {{0x63f88f8, 0xc0868062a0}, {0xc078780789, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0868062a0}, {0xc078780789, 0x23}, 0x0, 0x0}, {0x1d070ccb, 0xedb7a8350, 0x0}, {0x1d0714ff, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc09c1cf300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc09c1cf300, {0x63c8510, 0xc033578280}, 0xc07b0d7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0343c1620) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc033578280}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0343c1620) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808419 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc05aaa6630}, 0xc051d74900, 0xc07ae3a960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc05aaa6630}, 0xc051d74900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c49915b, 0x0, 0x0}, 0xc057ecaa10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c49915b, 0x0, 0x0}, 0xc057ecaa10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05aaa65a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05aaa65a0}, {0xc00b4cb3f8, 0x660089}, 0xc00b4cb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05aaa65a0}, {0x7265765f6970615f}, 0xc051d74700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05aaa6510}, 0xc051d74700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}, 0xc00ff18320, 0xc020e83cb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}, 0x3, 0xc059dfb300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b4cb980}}}, {0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}, 0xc00ff18320, 0xc059dfb300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05aaa6510}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05aaa6510}, {0x40, 0xc00b4cbb70}, 0xc00b4cbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05aaa6510}, {0x4e9d7a0, 0xc051d74700}, 0xc00ff18320, 0xc020e83cb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05aaa6510}, 0xc046d0bce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059e98fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059e98fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799210 [runnable]: sync.runtime_SemacquireMutex(0xc08b210690, 0x8f, 0xc08b210478) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x191c615, 0x7fe3205d5440, 0x1194e0b) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x2a, 0x2a, 0x33, 0xc08b210560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc07cef6900, {0xffffffffffffff15, 0xc08b2106c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc07cef6900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc07cef6900, {0xc01247d100, 0x32, 0x71}, {0xc0249ab000, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0968f1298, {0xc01247d100, 0x32, 0x71}, {0xc0249ab000, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0968efbc0, {0xc0249ab000, 0x74, 0x71}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0968efbc0, {0xc01247d100, 0x74, 0xc0249ab000}, {0xc0249ab000, 0xc0596b3ad0, 0x31}, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0968ef900, {0xc0249ab000, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc09fa1de28, {{0xc0596b3ad0, 0x31, 0xd0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06ab86c00, {{0xc02971ff50, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f398000, {0x63c85b8, 0xc0521d0120}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0521d0120}, {0x64a6560, 0xc06ab86c00}, {0xc02971ff50, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0521d0120}, {0x6466ed8, 0xc09fa1db80}, {0xc02971ff50, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04d8765b8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc047b03300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0207d4000, {0x63c85b8, 0xc0521d0120}, 0xc047b03300, 0xc05a454e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0207d4000, {0x63c85b8, 0xc0521d0120}, 0xc047b03300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0521d0090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0521d0090}, {0xc0036f68f0, 0x660089}, 0xc0036f68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0521d0090}, {0xc00102e4e0}, 0xc047b03200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0521d0030}, 0xc047b03200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc78ba0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812448 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc073ab2ed0}, 0xc06c743bc0, 0x1, 0xc04285d0a8, 0x519bbc, 0xc00de16420, 0xc0784b7620) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc02e545e60, {0x63c85b8, 0xc073ab2ed0}, 0xc05baefc20, 0xc04285d1a8, 0xde16420, 0x0, 0x1, 0xc00de16420, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc073ab2ed0}, 0xc00de16360, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc053bd8fc0, 0xb4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073ab2ed0}, 0x6352260, 0x36ce8338) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc073ab2ed0}, {0xc00de16240, {0x174301104be44f22, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc073ab2ed0}, 0xc02a137180, 0xc04285d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073ab2ed0}, 0x660089, {0xc00de16240, {0x174301104be44f22, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc073ab2ed0}, 0xc065f94d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104be44f22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104be44f22, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073ab2e10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073ab2e10}, {0xc04285f3f8, 0x660089}, 0xc04285f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073ab2e10}, {0xc0015221e0}, 0xc065f94b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073ab2d80}, 0xc065f94b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}, 0xc06d8c0340, 0xc02b06a9c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}, 0x3, 0xc04df40c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04285f980}}}, {0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}, 0xc06d8c0340, 0xc04df40c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073ab2d80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073ab2d80}, {0x40, 0xc04285fb70}, 0xc04285fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073ab2d80}, {0x4e9d7a0, 0xc065f94b00}, 0xc06d8c0340, 0xc02b06a9c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073ab2d80}, 0xc02423d6e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e9440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e9440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810284 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0210c0400, {0x63c85b8, 0xc074e1a690}, 0xc0b5d8af00, 0xc02e01b4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c0400, {0x63c85b8, 0xc074e1a690}, 0xc0b5d8af00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cf719b1, 0x0, 0x0}, 0xc044115710, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cf719b1, 0x0, 0x0}, 0xc044115710, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074e1a600}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074e1a600}, {0xc028f893f8, 0x660089}, 0xc028f89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074e1a600}, {0xc0015221e0}, 0xc0b5d8ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074e1a570}, 0xc0b5d8ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}, 0xc0267a65c0, 0xc07289abb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}, 0x3, 0xc01b5f4b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028f89980}}}, {0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}, 0xc0267a65c0, 0xc01b5f4b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc074e1a570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074e1a570}, {0x40, 0xc028f89b70}, 0xc028f89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc074e1a570}, {0x4e9d7a0, 0xc0b5d8ad00}, 0xc0267a65c0, 0xc07289abb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc074e1a570}, 0xc0983d0b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060f5b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060f5b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812776 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811152 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea3400, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc072d7be80, 0xc00e202800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc072d7be80, {0xc00e202800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc072d7be80, {0xc00e202800, 0x735, 0xc020b708c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03f55a090, {0xc00e202800, 0x7eda79, 0xc00e202805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc065e1d1a0, {0xc00e202800, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03c539650, {0xc00e202800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0137c13f8, {0x631b900, 0xc03c539650}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0137c1180, {0x7fe42ab59310, 0xc065e1d1a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0137c1180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0137c1180, {0xc08c3aa000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04c838828, {0xc08c3aa000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc048c51588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc048c51588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc048c51b60, {0x634aac0, 0xc048c51588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc048c51500, 0xc07603a5c0, 0xc083bc342b, 0xc083bc342d, {0x635bec0, 0xc016cdfa70}, 0xc000d6ae00, 0xc02bdb4ae8, 0xc083bc3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc048c51500, {0x63c85b8, 0xc0413ea600}, 0xc07603a5b0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0449f303b, 0x8}}, 0x0, 0xc0413ea5a0, 0xc0413ea5d0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0413ea600}, {0x643d810, 0xc065e1d1a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0473c53e0}, {0x643d810, 0xc065e1d1a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0473c53e0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807104 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc02f0407e0}, 0xc01fb444e0, 0x1, 0xc004d230a8, 0x519bbc, 0xc0b9b52940, 0xc06ec22e40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0ada1aa20, {0x63c85b8, 0xc02f0407e0}, 0xc0ada1aa68, 0xc004d231a8, 0x41, 0x60, 0xffffffffffffffff, 0xc0b9b52940, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc02f0407e0}, 0xc0b9b52700, {{{{0x0, 0x0}, {0xc00d58ad80, 0xc}}, {{0xc061e6dc20, 0xf}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02f0407e0}, 0x63521a0, 0x1b928fd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc02f0407e0}, {0xc07c1417a0, {0x1743010ffd857223, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc02f0407e0}, 0xc0668da960, 0xc004d239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02f0407e0}, 0x660089, {0xc07c1417a0, {0x1743010ffd857223, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc02f0407e0}, 0xc08d515600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffd857223, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffd857223, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f040750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f040750}, {0xc004d253f8, 0x660089}, 0xc004d25400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f040750}, {0xc0015221e0}, 0xc08d515400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f040630}, 0xc08d515400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}, 0xc06d002600, 0xc02d7e32d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}, 0x3, 0xc0594ae6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004d25980}}}, {0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}, 0xc06d002600, 0xc0594ae6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f040630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f040630}, {0x40, 0xc004d25b70}, 0xc004d25ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f040630}, {0x4e9d7a0, 0xc08d515400}, 0xc06d002600, 0xc02d7e32d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f040630}, 0xc051a1d260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0791cde60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0791cde60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795859 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x6e) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0787ea0f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0543c3500, {0xc0612a1d60, 0x32, 0x97}, {0xc02f456080, 0x32, 0x77}, 0x1, 0xa0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0787e9f88, {0xc0612a1d60, 0x1, 0x77}, {0xc02f456080, 0x2d7, 0xc082e62790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0787e9f88, {0xc0612a1d60, 0x32, 0x97}, {0xc02f456080, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0787e96c0, {0xc02f456080, 0x77, 0x97}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0787e96c0, {0xc0612a1d60, 0x77, 0xc02f456080}, {0xc02f456080, 0xc070625400, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0787e9400, {0xc02f456080, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06150a828, {{0xc070625400, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc052b22900, {{0xc033021260, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0574b4000, {0x63c85b8, 0xc062c8b320}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062c8b320}, {0x64a6560, 0xc052b22900}, {0xc033021260, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062c8b320}, {0x6466ed8, 0xc06150a580}, {0xc033021260, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02e6372f0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0787aa900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0201d5800, {0x63c85b8, 0xc062c8b320}, 0xc0787aa900, 0xc0afd0ef00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0201d5800, {0x63c85b8, 0xc062c8b320}, 0xc0787aa900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062c8b290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062c8b290}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062c8b290}, {0xc00102e4e0}, 0xc0787aa800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062c8b230}, 0xc0787aa800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc062c8ac60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799747 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc094eb8548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc094eb85b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc091b656f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc007345500, {0xc038fda7e0, 0x2b, 0x6d}, {0xc041dcb5e0, 0x2b, 0x6b}, 0x1, 0x56) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091b65588, {0xc038fda7e0, 0x1, 0x6b}, {0xc041dcb5e0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091b65588, {0xc038fda7e0, 0x2b, 0x6d}, {0xc041dcb5e0, 0x2b, 0x6b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091b64cc0, {0xc041dcb5e0, 0x6b, 0x6d}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091b64cc0, {0xc038fda7e0, 0x2b, 0xc041dcb5e0}, {0xc041dcb5e0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091b64a00, {0xc041dcb5e0, 0x2b, 0x6b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc060b46c08, {{0xc0313dd830, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0c7aa5c00, {{0xc0313dd830, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0541f5c00, {0x63c85b8, 0xc04f2a1d40}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04f2a1d40}, {0x64a6560, 0xc0c7aa5c00}, {0xc0313dd830, 0x70, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04f2a1d40}, {0x6466ed8, 0xc060b46c00}, {0xc0313dd830, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc045e69f08}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc014164d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0205cdc00, {0x63c85b8, 0xc04f2a1d40}, 0xc014164d00, 0xc0601d8140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cdc00, {0x63c85b8, 0xc04f2a1d40}, 0xc014164d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f2a1cb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f2a1cb0}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f2a1cb0}, {0xc00102e4e0}, 0xc014164c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f2a1c50}, 0xc014164c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc09ed74120}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219150687 [select, 1 minutes]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc058137200) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc03de822c0, {0x4cfe6e0, 0xc042800a40}, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc058136d80, 0xc0199d4db0, 0xc0199d4d80) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:688 +0x2f6 google.golang.org/grpc.(*clientStream).RecvMsg(0xc058136d80, {0x4cfe6e0, 0xc042800a40}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f github.com/cockroachdb/cockroach/pkg/util/tracing.(*tracingClientStream).RecvMsg(0xc021a9a720, {0x4cfe6e0, 0xc042800a40}) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:440 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc011ac8840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:111 +0x4c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x643dbd8, 0xc011ac8840}, 0xc00a7da980, 0xc000cb3540, {0x63c85b8, 0xc053f5b140}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:482 +0x63 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x63c85b8, 0xc053f5b140}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:497 +0x48 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 209251923 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4de08, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04ddbdf80, 0xc06dfdcc00, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04ddbdf80, {0xc06dfdcc00, 0x127b, 0x127b}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04ddbdf80, {0xc06dfdcc00, 0x127b, 0xc0540d5180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03102e8e0, {0xc06dfdcc00, 0x7eda79, 0xc06dfdcc05}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04a42c900, {0xc06dfdcc00, 0xc0c2794c60, 0x127b}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc072326d50, {0xc06dfdcc00, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0713725f8, {0x631b900, 0xc072326d50}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc071372380, {0x7fe42ab59310, 0xc04a42c900}, 0x127b) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc071372380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc071372380, {0xc0727b8000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02902db00, {0xc0727b8000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc078274088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc078274088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc078274660, {0x634aac0, 0xc078274088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc078274000, 0xc03f424380, 0xc05ecaf42b, 0xc05ecaf42d, {0x635bec0, 0xc06fb2f050}, 0xc000d6ae00, 0xc01d40e168, 0xc05ecaf429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc078274000, {0x63c85b8, 0xc079d2bbf0}, 0xc03f424370, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0006d7009, 0x8}}, 0x0, 0xc079d2bb90, 0xc079d2bbc0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc079d2bbf0}, {0x643d810, 0xc04a42c900}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc017be3e00}, {0x643d810, 0xc04a42c900}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc017be3e00}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802468 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc018fa0de0}, 0xc00a65bd40, 0x1, 0xc0602650a8, 0x519bbc, 0xc0365ab320, 0xc03a0618c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc018fa0de0}, 0x1e0, 0xc060265230, 0x365ab2c0, 0x0, 0x660069, 0xc0365ab320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc018fa0de0}, 0xc0365ab2c0, {{{{0x0, 0x0}, {0xc09ed7fb00, 0xc4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc018fa0de0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc018fa0de0}, {0xc07af91b00, {0x1743011020a18edb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc018fa0de0}, 0xc07a98adc0, 0xc0602659f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc018fa0de0}, 0x660089, {0xc07af91b00, {0x1743011020a18edb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc018fa0de0}, 0xc0b4406500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020a18edb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020a18edb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc018fa0d50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018fa0d50}, {0xc0602673f8, 0x660089}, 0xc060267400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc018fa0d50}, {0x0}, 0xc0b4406300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018fa0cc0}, 0xc0b4406300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}, 0xc054659cc0, 0xc01ff38078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}, 0x3, 0xc05287a340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc060267980}}}, {0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}, 0xc054659cc0, 0xc05287a340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc018fa0cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018fa0cc0}, {0x40, 0xc060267b70}, 0xc060267ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc018fa0cc0}, {0x4e9d7a0, 0xc0b4406300}, 0xc054659cc0, 0xc01ff38078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc018fa0cc0}, 0xc085942b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07af919e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07af919e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806911 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024c49040, {0x63c85b8, 0xc0931dce40}, 0xc038af9ef0, 0x1, 0x708017f0de986b, 0x37e11d600, 0xc048d5e060, 0xc062eda840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0931dce40}, 0x120, 0xc0049eb230, 0x48d5e000, 0x0, 0x660069, 0xc048d5e060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024c49040, {0x63c85b8, 0xc0931dce40}, 0xc048d5e000, {{{{0x0, 0x0}, {0xc0b4c8e480, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0931dce40}, 0x6352410, 0x4f454cd8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024c04f50, {0x63c85b8, 0xc0931dce40}, {0xc037fbfe60, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024c04f50, {0x63c85b8, 0xc0931dce40}, 0xc06d366f00, 0xc0049eb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0931dce40}, 0x660089, {0xc037fbfe60, {0x174301104d62dc8b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0248fa400, {0x63c85b8, 0xc0931dce40}, 0xc0791e6a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d62dc8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0931dcd80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0931dcd80}, {0xc0049ed3f8, 0x660089}, 0xc0049ed400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0931dcd80}, {0x40706fd906efbc51}, 0xc0791e6800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0931dccf0}, 0xc0791e6800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}, 0xc08fc48cc0, 0xc038af9e90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}, 0x3, 0xc0927a85c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0049ed980}}}, {0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}, 0xc08fc48cc0, 0xc0927a85c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0931dccf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0931dccf0}, {0x40, 0xc0049edb70}, 0xc0049edab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0931dccf0}, {0x4e9d7a0, 0xc0791e6800}, 0xc08fc48cc0, 0xc038af9e90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0931dccf0}, 0xc065d02660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037fbfd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037fbfd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807931 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bcd0140, {0x63c85b8, 0xc077516a50}, 0xc039d97f50, 0x1, 0x0, 0x1, 0xc073141020, 0xc075b9c060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077516a50}, 0x120, 0xc00eb4c728, 0x73140fc0, 0x0, 0x660069, 0xc073141020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bcd0140, {0x63c85b8, 0xc077516a50}, 0xc073140fc0, {{{{0x0, 0x0}, {0xc0906b9440, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077516a50}, 0x6352410, 0x32430ad8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bcc42d0, {0x63c85b8, 0xc077516a50}, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bcc42d0, {0x63c85b8, 0xc077516a50}, 0xc0239ea8c0, 0xc07c606ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077516a50}, 0x660089, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bcd2c00, {0x63c85b8, 0xc077516a50}, 0xc05156a900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0775169c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0775169c0}, {0xc07d2508f0, 0x660089}, 0xc07d2508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0775169c0}, {0xc00102e4e0}, 0xc05156a800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077516960}, 0xc05156a800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04e72e720}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803479 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc043d668d0}, 0xc08837d6c8, 0x1, 0x0, 0x1, 0xc046fae7e0, 0xc04810aba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc043d668d0}, 0x1e0, 0xc075924728, 0x46fae780, 0x0, 0x660069, 0xc046fae7e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc043d668d0}, 0xc046fae780, {{{{0x0, 0x0}, {0xc079beab40, 0x42}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc04d8af440, {0x63c85b8, 0xc043d668d0}, 0xc05c1f7950, 0x76114920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc043d668d0}, {0xc03b85e120, {0x174301102ce103ca, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc043d668d0}, 0xc07c637e00, 0xc076114ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc043d668d0}, 0x660089, {0xc03b85e120, {0x174301102ce103ca, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc043d668d0}, 0xc05a793000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ce103ca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ce103ca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043d66840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043d66840}, {0xc0492c88f0, 0x660089}, 0xc0492c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043d66840}, {0xc00102e4e0}, 0xc05a792f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043d667e0}, 0xc05a792f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0190e3ec0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812739 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803611 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc021a2a3f0}, 0xc00a65b038, 0x1, 0xc075b7b0a8, 0x519bbc, 0xc048268f60, 0xc047515320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc021a2a3f0}, 0x1e0, 0xc075b7b230, 0x48268f00, 0x0, 0x660069, 0xc048268f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc021a2a3f0}, 0xc048268f00, {{{{0x0, 0x0}, {0xc03d108900, 0x44}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc021a2a3f0}, 0x63521a0, 0x34330728) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc021a2a3f0}, {0xc041668900, {0x174301102e4959b8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc021a2a3f0}, 0xc0875c8dc0, 0xc075b7b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc021a2a3f0}, 0x660089, {0xc041668900, {0x174301102e4959b8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc021a2a3f0}, 0xc05c1ff100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e4959b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e4959b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021a2a360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021a2a360}, {0xc075b7d3f8, 0x660089}, 0xc075b7d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021a2a360}, {0xc07c93f580}, 0xc05c1fef00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021a2a2a0}, 0xc05c1fef00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}, 0xc03d18a140, 0xc030069698) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}, 0x3, 0xc08689c740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075b7d980}}}, {0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}, 0xc03d18a140, 0xc08689c740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc021a2a2a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021a2a2a0}, {0x40, 0xc075b7db70}, 0xc075b7dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc021a2a2a0}, {0x4e9d7a0, 0xc05c1fef00}, 0xc03d18a140, 0xc030069698) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc021a2a2a0}, 0xc0306eac00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041d2c240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041d2c240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802005 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc09694c548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc09694c5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0787e0cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc07dead500, {0xc024e42a00, 0x2b, 0x74}, {0xc041dcb3b0, 0x2b, 0x6b}, 0x1, 0xc) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0787e0b88, {0xc024e42a00, 0x1, 0x6b}, {0xc041dcb3b0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0787e0b88, {0xc024e42a00, 0x2b, 0x74}, {0xc041dcb3b0, 0x2b, 0x6b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0787e02c0, {0xc041dcb3b0, 0x6b, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0787e02c0, {0xc024e42a00, 0x2b, 0xc041dcb3b0}, {0xc041dcb3b0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0787e0000, {0xc041dcb3b0, 0x2b, 0x6b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0197d4b08, {{0xc06dc2a420, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc00436f400, {{0xc06dc2a420, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0553f9c00, {0x63c85b8, 0xc041cebe60}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc041cebe60}, {0x64a6560, 0xc00436f400}, {0xc06dc2a420, 0x26c79, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc041cebe60}, {0x6466ed8, 0xc0197d4b00}, {0xc06dc2a420, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9b0c8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01eb22d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ebb000, {0x63c85b8, 0xc041cebe60}, 0xc01eb22d00, 0xc07b9dfa40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ebb000, {0x63c85b8, 0xc041cebe60}, 0xc01eb22d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041cebdd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041cebdd0}, {0xc0076c68f0, 0x660089}, 0xc0076c68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041cebdd0}, {0xc00102e4e0}, 0xc01eb22c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041cebd70}, 0xc01eb22c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618cb290}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806339 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc089a2ecf0}, 0xc042b0db00, 0xc0721c9ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc089a2ecf0}, 0xc042b0db00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d6ced7d, 0x0, 0x0}, 0xc0767e0af0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d6ced7d, 0x0, 0x0}, 0xc0767e0af0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc089a2ec60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc089a2ec60}, {0xc00778d3f8, 0x660089}, 0xc00778d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc089a2ec60}, {0x7fe428727d58}, 0xc0049b5800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc089a2ebd0}, 0xc0049b5800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}, 0xc077ca44a0, 0xc0907c4048) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}, 0x3, 0xc01b101380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00778d980}}}, {0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}, 0xc077ca44a0, 0xc01b101380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc089a2ebd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc089a2ebd0}, {0x40, 0xc00778db70}, 0xc00778dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc089a2ebd0}, {0x4e9d7a0, 0xc0049b5800}, 0xc077ca44a0, 0xc0907c4048) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc089a2ebd0}, 0xc0308d01e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04e3e9320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04e3e9320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812528 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 214101230 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1ad0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06e41d780, 0xc0a8ac0a00, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06e41d780, {0xc0a8ac0a00, 0x138b, 0x138b}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06e41d780, {0xc0a8ac0a00, 0x138b, 0xc05537c3c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc015789388, {0xc0a8ac0a00, 0x80, 0x7f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc057832fc0, {0xc0a8ac0a00, 0x3, 0x138b}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02132a138, {0xc0a8ac0a00, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc05224b078, {0x631b900, 0xc02132a138}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc05224ae00, {0x7fe42ab59310, 0xc057832fc0}, 0x138b) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc05224ae00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc05224ae00, {0xc055948000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06258a0c0, {0xc055948000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc042eb2b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc042eb2b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc042eb30e0, {0x634aac0, 0xc042eb2b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc042eb2a80, 0xc05a57c190, 0xc07b0db42b, 0xc07b0db42d, {0x635bec0, 0xc0667ac090}, 0xc000d6ae00, 0xc04efc2888, 0xc07b0db429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc042eb2a80, {0x63c85b8, 0xc0688166c0}, 0xc05a57c180, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc077a2604a, 0x8}}, 0x0, 0xc068816660, 0xc068816690, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0688166c0}, {0x643d810, 0xc057832fc0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc029e1ef90}, {0x643d810, 0xc057832fc0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc029e1ef90}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799211 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc06ffa4548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc06ffa45b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc09351fbf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04c53dc00, {0xc01d5863c0, 0x2b, 0x3e}, {0xc0430c9a40, 0x2b, 0x65}, 0x1, 0xfb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc09351fa88, {0xc01d5863c0, 0x1, 0x65}, {0xc0430c9a40, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09351fa88, {0xc01d5863c0, 0x2b, 0x3e}, {0xc0430c9a40, 0x2b, 0x65}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09351f1c0, {0xc0430c9a40, 0x65, 0x3e}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09351f1c0, {0xc01d5863c0, 0x2b, 0xc0430c9a40}, {0xc0430c9a40, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09351ef00, {0xc0430c9a40, 0x2b, 0x65}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0428fb188, {{0xc09be3e6f0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0487bdf00, {{0xc09be3e6f0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0414f2000, {0x63c85b8, 0xc0ae7694d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0ae7694d0}, {0x64a6560, 0xc0487bdf00}, {0xc09be3e6f0, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0ae7694d0}, {0x6466ed8, 0xc0428fb180}, {0xc09be3e6f0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c019458}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04a6b4000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0207d4c00, {0x63c85b8, 0xc0ae7694d0}, 0xc04a6b4000, 0xc0307a6a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0207d4c00, {0x63c85b8, 0xc0ae7694d0}, 0xc04a6b4000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0ae769440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ae769440}, {0xc0032fa8f0, 0x660089}, 0xc0032fa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0ae769440}, {0xc00102e4e0}, 0xc0295d8b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0ae7693e0}, 0xc0295d8b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc78c00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810166 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e72bc00, {0x63c85b8, 0xc02e33cf00}, 0xc0292c4a00, 0xc05953f400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc02e33cf00}, 0xc0292c4a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106b991fd8, 0x0, 0x0}, 0xc053007210, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106b991fd8, 0x0, 0x0}, 0xc053007210, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02e33ce70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02e33ce70}, {0xc0881372a8, 0x660089}, 0xc0881372b0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02e33ce70}, {0xc00102e4e0}, 0xc0292c4800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02e33ce10}, 0xc0292c4800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8580, 0xc046e4a4e0}, 0x0, 0xc074017180) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c8580, 0xc046e4a4e0}, 0x63c8580) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftLogQueue).process(0xc003629840, {0x63c8580, 0xc046e4a4e0}, 0xc02e72bc00, {0xc046e4a4e0, 0x65f500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_log_queue.go:740 +0x87c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica.func1({0x63c8580, 0xc046e4a4e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:977 +0x26e github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02e33cc60}, {0xc078af1710, 0x23}, 0xdf8475800, 0xc0ad83de28) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica(0xc00d506f00, {0x63c85b8, 0xc02e33cc30}, {0x6463f50, 0xc02e72bc00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:936 +0x3f5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2.1({0x63c85b8, 0xc02e33cc30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:855 +0xde github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808410 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc055acae50, 0x6) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc033763500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc033763b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc041208600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc041208600, {0x63c8510, 0xc055acae80}, 0xc082d17dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04325ee00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc055acae80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04325ee00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812026 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc01ed92990}, 0xc083507300, 0xc07c0b8780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc01ed92990}, 0xc083507300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107dc2e172, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107dc2e172, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01ed92900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ed92900}, {0xc032eb33f8, 0x660089}, 0xc032eb3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01ed92900}, {0xc0015221e0}, 0xc083507100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01ed92870}, 0xc083507100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}, 0xc06b235ae0, 0xc03c1ea018) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}, 0x3, 0xc063be5500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032eb3980}}}, {0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}, 0xc06b235ae0, 0xc063be5500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01ed92870}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ed92870}, {0x40, 0xc032eb3b70}, 0xc032eb3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01ed92870}, {0x4e9d7a0, 0xc083507100}, 0xc06b235ae0, 0xc03c1ea018) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01ed92870}, 0xc0314ba3c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01b225680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01b225680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808497 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc028e0f740}, 0xc03592d350, 0x1, 0xc00da330a8, 0x519bbc, 0xc06653eae0, 0xc04d8b9e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01c567e00, {0x63c85b8, 0xc028e0f740}, 0xc0b72f06c0, 0xc00da331a8, 0x6653eae0, 0x0, 0x0, 0xc06653eae0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc028e0f740}, 0xc06653ea20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc090e39200, 0x7a}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc028e0f740}, 0x6352260, 0x32431a90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc028e0f740}, {0xc06653e900, {0x1743011024123859, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc028e0f740}, 0xc033dbadc0, 0xc00da339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc028e0f740}, 0x660089, {0xc06653e900, {0x1743011024123859, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc028e0f740}, 0xc05be2f800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024123859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024123859, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc028e0f680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028e0f680}, {0xc00da353f8, 0x660089}, 0xc00da35400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc028e0f680}, {0x29}, 0xc05be2f600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc028e0f5f0}, 0xc05be2f600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}, 0xc029a8e400, 0xc043918f90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}, 0x3, 0xc09f0c0b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00da35980}}}, {0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}, 0xc029a8e400, 0xc09f0c0b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc028e0f5f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028e0f5f0}, {0x40, 0xc00da35b70}, 0xc00da35ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc028e0f5f0}, {0x4e9d7a0, 0xc05be2f600}, 0xc029a8e400, 0xc043918f90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc028e0f5f0}, 0xc020e3ef00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06653e7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06653e7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805645 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc022e847e0}, 0xc03a365860, 0x1, 0xc005bd30a8, 0x519bbc, 0xc08bd84600, 0xc067461020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc022e847e0}, 0x120, 0xc005bd3230, 0x8bd845a0, 0x0, 0x660069, 0xc08bd84600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc022e847e0}, 0xc08bd845a0, {{{{0x0, 0x0}, {0xc0670db440, 0xf7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc022e847e0}, 0x6352410, 0x1cdd97f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc022e847e0}, {0xc08bd84480, {0x174301104a33b713, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc022e847e0}, 0xc06e4360a0, 0xc005bd39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022e847e0}, 0x660089, {0xc08bd84480, {0x174301104a33b713, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc022e847e0}, 0xc063448e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104a33b713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104a33b713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022e84750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022e84750}, {0xc005bd53f8, 0x660089}, 0xc005bd5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022e84750}, {0xb1822aff40e6533}, 0xc063448c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022e84660}, 0xc063448c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}, 0xc06deb86a0, 0xc07d344de0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}, 0x3, 0xc0639a3500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005bd5980}}}, {0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}, 0xc06deb86a0, 0xc0639a3500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022e84660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022e84660}, {0x40, 0xc005bd5b70}, 0xc005bd5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022e84660}, {0x4e9d7a0, 0xc063448c00}, 0xc06deb86a0, 0xc07d344de0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022e84660}, 0xc02ecac780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08bd84240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08bd84240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219792259 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0b885d440}, 0xc02b908cb0, {0x646cc48, 0xc0b885d4a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0b885d440}, 0xedb7a834e, 0xc0679db680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0b885d440}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0152628d0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc02b908ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0152628d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809100 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc040c5e690}, 0xc056516b58, 0x1, 0xc0235e10a8, 0x519bbc, 0xc050f9e420, 0xc076f2c060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc040c5e690}, 0x1e0, 0xc0235e1230, 0x50f9e3c0, 0x0, 0x660069, 0xc050f9e420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc040c5e690}, 0xc050f9e3c0, {{{{0x0, 0x0}, {0xc0968a8240, 0x128}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc01dd253e0, {0x63c85b8, 0xc040c5e690}, 0xc0201ef690, 0x235e1428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc040c5e690}, {0xc04e3f5d40, {0x17430110619e3176, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc040c5e690}, 0xc0314e00a0, 0xc0235e19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc040c5e690}, 0x660089, {0xc04e3f5d40, {0x17430110619e3176, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc040c5e690}, 0xc056dd1100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110619e3176, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110619e3176, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc040c5e3f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc040c5e3f0}, {0xc0235e33f8, 0x660089}, 0xc0235e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc040c5e3f0}, {0xc0015221e0}, 0xc056dd0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc040c5e330}, 0xc056dd0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}, 0xc0746fff20, 0xc056517188) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}, 0x3, 0xc072990f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0235e3980}}}, {0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}, 0xc0746fff20, 0xc072990f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc040c5e330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc040c5e330}, {0x40, 0xc0235e3b70}, 0xc0235e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc040c5e330}, {0x4e9d7a0, 0xc056dd0f00}, 0xc0746fff20, 0xc056517188) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc040c5e330}, 0xc038a09c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc087325c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc087325c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800380 [runnable]: sync.runtime_SemacquireMutex(0xc091446690, 0x8f, 0x7fd4b7a70a40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc091446538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc091446560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0131c2900, {0xffffffffffffff15, 0xc0914466c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0131c2900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0131c2900, {0xc025cb9a00, 0x32, 0x7a}, {0xc02e184900, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092dab298, {0xc025cb9a00, 0x32, 0x7a}, {0xc02e184900, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092da9bc0, {0xc02e184900, 0x74, 0x7a}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092da9bc0, {0xc025cb9a00, 0x74, 0xc02e184900}, {0xc02e184900, 0xc00a796e60, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092da9900, {0xc02e184900, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0423262a8, {{0xc00a796e60, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05fafd600, {{0xc0527f0180, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc081b47000, {0x63c85b8, 0xc0780ae6f0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0780ae6f0}, {0x64a6560, 0xc05fafd600}, {0xc0527f0180, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0780ae6f0}, {0x6466ed8, 0xc042326000}, {0xc0527f0180, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c5ec3f0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0488acf00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020abb000, {0x63c85b8, 0xc0780ae6f0}, 0xc0488acf00, 0xc03b365ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020abb000, {0x63c85b8, 0xc0780ae6f0}, 0xc0488acf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0780ae660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0780ae660}, {0xc0933e48f0, 0x660089}, 0xc0933e48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0780ae660}, {0xc00102e4e0}, 0xc0488ace00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0780ae600}, 0xc0488ace00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b770}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807581 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc06e374840}, 0xc03bfb3740, 0x1, 0xc00f59f0a8, 0x519bbc, 0xc073d85a40, 0xc09433acc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06e374840}, 0x120, 0xc00f59f230, 0x73d859e0, 0x0, 0x660069, 0xc073d85a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc06e374840}, 0xc073d859e0, {{{{0x0, 0x0}, {0xc05c03a900, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06e374840}, 0x6352410, 0x1906edc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc06e374840}, {0xc073d857a0, {0x174301105981709e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc06e374840}, 0xc043be9a40, 0xc00f59f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06e374840}, 0x660089, {0xc073d857a0, {0x174301105981709e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc06e374840}, 0xc05c61a100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105981709e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105981709e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e3747b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e3747b0}, {0xc00f5a13f8, 0x660089}, 0xc00f5a1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e3747b0}, {0x7fe3317a4b68}, 0xc04acb3700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e374720}, 0xc04acb3700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}, 0xc03d581160, 0xc04ddb22d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}, 0x3, 0xc0762ecc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f5a1980}}}, {0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}, 0xc03d581160, 0xc0762ecc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06e374720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e374720}, {0x40, 0xc00f5a1b70}, 0xc00f5a1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06e374720}, {0x4e9d7a0, 0xc04acb3700}, 0xc03d581160, 0xc04ddb22d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06e374720}, 0xc0424ce1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc073d84fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc073d84fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802800 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc02c836780}, 0xc0414c2ca8, 0x1, 0xc064f6d0a8, 0x519bbc, 0xc03b7ed8c0, 0xc03b578600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02c836780}, 0x1e0, 0xc064f6d230, 0x3b7ed860, 0x0, 0x660069, 0xc03b7ed8c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc02c836780}, 0xc03b7ed860, {{{{0x0, 0x0}, {0xc05ab867e0, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc02c836780}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc02c836780}, {0xc048b2cea0, {0x1743011020ee152a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc02c836780}, 0xc06b74c500, 0xc064f6d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02c836780}, 0x660089, {0xc048b2cea0, {0x1743011020ee152a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc02c836780}, 0xc0b1d5ff00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020ee152a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020ee152a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02c8366c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c8366c0}, {0xc064f6f3f8, 0x660089}, 0xc064f6f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02c8366c0}, {0x7fe3c0104398}, 0xc0b1d5fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02c836630}, 0xc0b1d5fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}, 0xc0763f2f60, 0xc02bf28648) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}, 0x3, 0xc0156de600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc064f6f980}}}, {0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}, 0xc0763f2f60, 0xc0156de600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02c836630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c836630}, {0x40, 0xc064f6fb70}, 0xc064f6fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02c836630}, {0x4e9d7a0, 0xc0b1d5fd00}, 0xc0763f2f60, 0xc02bf28648) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02c836630}, 0xc05af55020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc048b2cd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc048b2cd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809788 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc081e65620}, 0xc01227acd8, 0x1, 0xc0137870a8, 0x519bbc, 0xc03edbe7e0, 0xc0347c1c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc081e65620}, 0x1e0, 0xc013787230, 0x3edbe780, 0x0, 0x660069, 0xc03edbe7e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc081e65620}, 0xc03edbe780, {{{{0x0, 0x0}, {0xc054807680, 0xea}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc081e65620}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc081e65620}, {0xc0997827e0, {0x1743011066fd62a1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc081e65620}, 0xc07cfa6820, 0xc0137879f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc081e65620}, 0x660089, {0xc0997827e0, {0x1743011066fd62a1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc081e65620}, 0xc082f59e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066fd62a1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066fd62a1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc081e65530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc081e65530}, {0xc0137893f8, 0x660089}, 0xc013789400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc081e65530}, {0x6845566d534a6552}, 0xc082f59c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc081e653e0}, 0xc082f59c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}, 0xc03248a840, 0xc02132a4c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}, 0x3, 0xc05b24adc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc013789980}}}, {0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}, 0xc03248a840, 0xc05b24adc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc081e653e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc081e653e0}, {0x40, 0xc013789b70}, 0xc013789ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc081e653e0}, {0x4e9d7a0, 0xc082f59c00}, 0xc03248a840, 0xc02132a4c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc081e653e0}, 0xc0a79da8a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0997826c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0997826c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219150691 [runnable]: internal/poll.runtime_pollWait(0x7fdb124bce50, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0481b3f80, 0xc0b5afc000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0481b3f80, {0xc0b5afc000, 0xf036, 0xf036}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0481b3f80, {0xc0b5afc000, 0xf036, 0xc04aad8500}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06b2453d0, {0xc0b5afc000, 0x7eda79, 0xc0b5afc005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc024db5c80, {0xc0b5afc000, 0x0, 0xf036}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0257a26d8, {0xc0b5afc000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0a0b03af8, {0x631b900, 0xc0257a26d8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0a0b03880, {0x7fe42ab59310, 0xc024db5c80}, 0xa5294a52) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0a0b03880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0a0b03880, {0xc0ab5c6000, 0x8000, 0x12553010001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc073504600, {0xc0473437e0, 0x9, 0xc0257a26c0}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc073504600}, {0xc0473437e0, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0473437e0, 0x9, 0xc000118060}, {0x631b600, 0xc073504600}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0473437a0) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc06cd01860, 0x10bed2f, 0x63c85b8) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc06cd01860}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 219812774 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801033 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc06087a990}, 0xc063c5f458, 0x1, 0xc038cd10a8, 0x519bbc, 0xc0330c0f60, 0xc027f5d6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06087a990}, 0x1e0, 0xc038cd1230, 0x330c0f00, 0x0, 0x660069, 0xc0330c0f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc06087a990}, 0xc0330c0f00, {{{{0x0, 0x0}, {0xc04bea98c0, 0x3a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06087a990}, 0x63521a0, 0x6541e2d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc06087a990}, {0xc01aad2d80, {0x174301100fab231d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc06087a990}, 0xc0364b20a0, 0xc038cd19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06087a990}, 0x660089, {0xc01aad2d80, {0x174301100fab231d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc06087a990}, 0xc034fadc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100fab231d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100fab231d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06087a900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06087a900}, {0xc038cd33f8, 0x660089}, 0xc038cd3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06087a900}, {0xc0015221e0}, 0xc034fada00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06087a870}, 0xc034fada00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}, 0xc03b73ae20, 0xc06ef88c18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}, 0x3, 0xc065eac480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc038cd3980}}}, {0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}, 0xc03b73ae20, 0xc065eac480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06087a870}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06087a870}, {0x40, 0xc038cd3b70}, 0xc038cd3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06087a870}, {0x4e9d7a0, 0xc034fada00}, 0xc03b73ae20, 0xc06ef88c18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06087a870}, 0xc0abcef0e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01aad2b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01aad2b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804980 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc048382d80}, 0xc0483f7260, 0x1, 0x708017d8977c3d, 0x37e11d600, 0xc06f85d4a0, 0xc076d963c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc048382d80}, 0x120, 0xc07be51230, 0x6f85d440, 0xc0545a1ba0, 0x18, 0xc06f85d4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc048382d80}, 0xc06f85d440, {{{{0x0, 0x0}, {0xc09298fd40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc048382d80}, 0x6352410, 0x1b25d2e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc048382d80}, {0xc06f85d320, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc048382d80}, 0xc043412820, 0xc07be519f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc048382d80}, 0x660089, {0xc06f85d320, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc048382d80}, 0xc011f4e600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048382cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048382cf0}, {0xc07be533f8, 0x660089}, 0xc07be53400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048382cf0}, {0x4d939c}, 0xc0878b1e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048382c60}, 0xc0878b1e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}, 0xc0235c5420, 0xc0483f7230) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}, 0x3, 0xc01ae26c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07be53980}}}, {0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}, 0xc0235c5420, 0xc01ae26c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048382c60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048382c60}, {0x40, 0xc07be53b70}, 0xc07be53ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}, 0x203006, 0x203006) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048382c60}, {0x4e9d7a0, 0xc0878b1e00}, 0xc0235c5420, 0xc0483f7230) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048382c60}, 0xc027dfaa80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85d200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85d200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802201 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc066fbeba0}, 0xc070822c30, 0x1, 0xc01ba3d0a8, 0x519bbc, 0xc020daca80, 0xc0874b0f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc066fbeba0}, 0x120, 0xc01ba3d230, 0x20daca20, 0x0, 0x660069, 0xc020daca80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc066fbeba0}, 0xc020daca20, {{{{0x0, 0x0}, {0xc088ff3c20, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc066fbeba0}, 0x6352410, 0x6541fb38) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc066fbeba0}, {0xc020dac7e0, {0x1743011020aa779b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc066fbeba0}, 0xc085729e00, 0xc01ba3d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066fbeba0}, 0x660089, {0xc020dac7e0, {0x1743011020aa779b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc066fbeba0}, 0xc06bd2d100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020aa779b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020aa779b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066fbe960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066fbe960}, {0xc01ba3f3f8, 0x660089}, 0xc01ba3f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066fbe960}, {0xc01ba3f490}, 0xc06bd2cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066fbe6c0}, 0xc06bd2cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}, 0xc05482d1e0, 0xc0776ad2f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}, 0x3, 0xc022d7e0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01ba3f980}}}, {0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}, 0xc05482d1e0, 0xc022d7e0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066fbe6c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066fbe6c0}, {0x40, 0xc01ba3fb70}, 0xc01ba3fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066fbe6c0}, {0x4e9d7a0, 0xc06bd2cf00}, 0xc05482d1e0, 0xc0776ad2f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066fbe6c0}, 0xc05984e000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc018fe1e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc018fe1e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812772 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808130 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc018d6a990}, 0xc020c825a0, 0x1, 0x0, 0x1, 0xc07c67fe60, 0xc07c67e240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc018d6a990}, 0x1e0, 0xc007560728, 0x7c67fe00, 0x0, 0x660069, 0xc07c67fe60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc018d6a990}, 0xc07c67fe00, {{{{0x0, 0x0}, {0xc075bc1680, 0x46}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc032b4d740, {0x63c85b8, 0xc018d6a990}, 0xc0022ebd70, 0x58160920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc018d6a990}, {0xc07571e5a0, {0x1743011057a24815, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc018d6a990}, 0xc06fb2be00, 0xc058160ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc018d6a990}, 0x660089, {0xc07571e5a0, {0x1743011057a24815, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc018d6a990}, 0xc02171d200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc018d6a900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018d6a900}, {0xc09362e8f0, 0x660089}, 0xc09362e8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc018d6a900}, {0xc00102e4e0}, 0xc02171d000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018d6a600}, 0xc02171d000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04bcf5bf0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800724 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc077c10090}, 0xc05a388df8, 0x1, 0xc0721e9f20, 0x519bbc, 0xc088a5c5a0, 0xc0937329c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc077c10090}, 0xc0, 0xc0721ea0a8, 0x88a5c540, 0x0, 0x660069, 0xc088a5c5a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc077c10090}, 0xc088a5c540, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc077c10090}, 0x63524a0, 0x3a07f5a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc077c10090}, {0x0, {0x174301105698445f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc077c10090}, 0xc0652bb4a0, 0xc0721ea868) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077c10090}, 0x660089, {0x0, {0x174301105698445f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc077c10090}, 0xc089079900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077c10000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077c10000}, {0xc0721ec270, 0x660089}, 0xc0721ec278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077c10000}, {0xc0590a3bf0}, 0xc089079800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0119b7e90}, 0xc089079800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x174301105698445f, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc0218416b0}, 0x0, 0xc0944cc580) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc0218416b0}, 0x66040f) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc019919860, {0x63c85b8, 0xc0218416b0}, {{0xaf, 0x47, 0xb9, 0x42, 0xb2, 0x8d, 0x49, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:922 +0x1b0 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc019919860, {0x63c85b8, 0xc0218416b0}, 0xc02958e780, 0xc070b3d420, 0xc04eee8190) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:610 +0xd4a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc0218416b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc0590a3da0}, {{0xc0609e8bc0, 0x24, 0x24}}, 0xc0721ef380) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc019919860, {0x63c85b8, 0xc0590a3da0}, 0xc02958e780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0301af310, {0x63c85b8, 0xc0590a3da0}, {0x0, {0x1743011015edf56a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0590a3da0}, 0xc049d0caa0, 0xc0308909d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0590a3da0}, 0xc049d0caa0, {0x0, {0x1743011015edf56a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0590a3da0}, 0xc03cbb5300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011015edf56a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011015edf56a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011015edf56a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0590a3c80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0590a3c80}, {0xc047d893f8, 0x660089}, 0xc047d89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0590a3c80}, {0x31644c6c55576e4e}, 0xc03cbb5100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0590a3bf0}, 0xc03cbb5100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}, 0xc042912cc0, 0xc0211f9500) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}, 0x3, 0xc075b3b040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc047d89980}}}, {0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}, 0xc042912cc0, 0xc075b3b040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0590a3bf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0590a3bf0}, {0x40, 0xc047d89b70}, 0xc047d89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0590a3bf0}, {0x4e9d7a0, 0xc03cbb5100}, 0xc042912cc0, 0xc0211f9500) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0590a3bf0}, 0xc0b5c78480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06f08afc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06f08afc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804666 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaffb3a10, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0619de880, 0xc053c58000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0619de880, {0xc053c58000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0619de880, {0xc053c58000, 0x735, 0xc08d7fd180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0473ddd00, {0xc053c58000, 0x7eda79, 0xc053c58005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06d0e8240, {0xc053c58000, 0x6, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04a6d9a28, {0xc053c58000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04bbcde78, {0x631b900, 0xc04a6d9a28}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04bbcdc00, {0x7fe42ab59310, 0xc06d0e8240}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04bbcdc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04bbcdc00, {0xc0b024b000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc07820aa98, {0xc0b024b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc047b4b588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc047b4b588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc047b4bb60, {0x634aac0, 0xc047b4b588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc047b4b500, 0xc0773961f0, 0xc08204b42b, 0xc08204b42d, {0x635bec0, 0xc04da1cfc0}, 0xc000d6ae00, 0xc03018a248, 0xc08204b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc047b4b500, {0x63c85b8, 0xc0761b3770}, 0xc0773961e0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc02723d038, 0x8}}, 0x0, 0xc0761b36b0, 0xc0761b36e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0761b3770}, {0x643d810, 0xc06d0e8240}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04ecc49c0}, {0x643d810, 0xc06d0e8240}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04ecc49c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810222 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc062110bd0}, 0xc031b69d40, 0x1, 0x0, 0x1, 0xc00ce49ec0, 0xc00bc8fa40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc062110bd0}, 0x120, 0xc01aaf8728, 0xce49e60, 0x0, 0x660069, 0xc00ce49ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc062110bd0}, 0xc00ce49e60, {{{{0x0, 0x0}, {0xc07d47d9e0, 0x1b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc062110bd0}, 0x6352410, 0x6b2457f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc062110bd0}, {0xc00ce48b40, {0x174301106aa0ffb2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc062110bd0}, 0xc0463c1680, 0xc078182ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc062110bd0}, 0x660089, {0xc00ce48b40, {0x174301106aa0ffb2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc062110bd0}, 0xc02c4f1700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106aa0ffb2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106aa0ffb2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062110b40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062110b40}, {0xc0936928f0, 0x660089}, 0xc0936928f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062110b40}, {0xc00102e4e0}, 0xc02c4f1600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062110ae0}, 0xc02c4f1600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0501bbe00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219731773 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af2a158, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06bbdd880, 0xc011ae9800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06bbdd880, {0xc011ae9800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06bbdd880, {0xc011ae9800, 0x735, 0xc07299d7c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0538fb1a0, {0xc011ae9800, 0x7eda79, 0xc011ae9805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc073747da0, {0xc011ae9800, 0x3, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02c29d170, {0xc011ae9800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0849ba5f8, {0x631b900, 0xc02c29d170}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0849ba380, {0x7fe42ab59310, 0xc073747da0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0849ba380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0849ba380, {0xc04ad48000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc075a05e00, {0xc04ad48000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc072677588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc072677588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc072677b60, {0x634aac0, 0xc072677588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc072677500, 0xc07d3915a0, 0xc079d8d42b, 0xc079d8d42d, {0x635bec0, 0xc039189a70}, 0xc000d6ae00, 0xc038179278, 0xc079d8d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc072677500, {0x63c85b8, 0xc03a74c870}, 0xc07d391590, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09ae3e038, 0x8}}, 0x0, 0xc03a74c810, 0xc03a74c840, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03a74c870}, {0x643d810, 0xc073747da0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06d1f1ec0}, {0x643d810, 0xc073747da0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06d1f1ec0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811920 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035040, {0x63c85b8, 0xc02178d320}, 0xc0182a9f80, 0x1, 0xc03b7096d8, 0x519bbc, 0xc0708b4d20, 0xc09d10d2c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc042ec4480, {0x63c85b8, 0xc02178d320}, 0xc0708b4d20, 0xc03b7097d8, 0x41, 0x60, 0x7, 0xc0708b4d20, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035040, {0x63c85b8, 0xc02178d320}, 0xc0708b4c60, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc077e2f680, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02178d320}, 0x6352260, 0x3c90c500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd3b0, {0x63c85b8, 0xc02178d320}, {0xc0708b4b40, {0x174301104a2798a9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd3b0, {0x63c85b8, 0xc02178d320}, 0xc03ac41360, 0xc03b70a020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02178d320}, 0x660089, {0xc0708b4b40, {0x174301104a2798a9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc02178d320}, 0xc02664eb00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104a2798a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104a2798a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02178cf30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02178cf30}, {0xc06f205a28, 0x660089}, 0xc06f205a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02178cf30}, {0xc00102e4e0}, 0xc02664e900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02178cc60}, 0xc02664e900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0715d4850, {0x63c85b8, 0xc02178c630}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0715d4850, {0x63c85b8, 0xc02178c630}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc02178c630}, {0x4f67ad2, 0x6446ff8}, 0xc028b33fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219775621 [runnable]: internal/poll.runtime_pollWait(0x7fe38e3e0220, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc071c7de00, 0xc086fec000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc071c7de00, {0xc086fec000, 0x1855, 0x1855}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc071c7de00, {0xc086fec000, 0x1855, 0xc08f784140}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03fdabd50, {0xc086fec000, 0x7eda79, 0xc086fec005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc040d55c80, {0xc086fec000, 0x7, 0x1855}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc021b9b368, {0xc086fec000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04fc7e278, {0x631b900, 0xc021b9b368}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04fc7e000, {0x7fe42ab59310, 0xc040d55c80}, 0x60) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04fc7e000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04fc7e000, {0xc035a0c000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04676e6a8, {0xc035a0c000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06d913588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06d913588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06d913b60, {0x634aac0, 0xc06d913588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06d913500, 0xc05a8bd110, 0xc083cf142b, 0xc083cf142d, {0x635bec0, 0xc0575955f0}, 0xc000d6ae00, 0xc02ab86d28, 0xc083cf1429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06d913500, {0x63c85b8, 0xc043009a10}, 0xc05a8bd100, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc03f63e037, 0x8}}, 0x0, 0xc043009980, 0xc0430099e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc043009a10}, {0x643d810, 0xc040d55c80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0247bfd70}, {0x643d810, 0xc040d55c80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0247bfd70}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812574 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806559 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021d552c0, {0x63c85b8, 0xc08d298fc0}, 0xc0745a17a0, 0x1, 0x0, 0x1, 0xc00d54ba40, 0xc0d08ae2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08d298fc0}, 0x120, 0xc00519a728, 0xd54b9e0, 0x0, 0x660069, 0xc00d54ba40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021d552c0, {0x63c85b8, 0xc08d298fc0}, 0xc00d54b9e0, {{{{0x0, 0x0}, {0xc049fd7e60, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08d298fc0}, 0x6352410, 0x4207f6b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3cbe0, {0x63c85b8, 0xc08d298fc0}, {0xc065cddd40, {0x174301104edebd10, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3cbe0, {0x63c85b8, 0xc08d298fc0}, 0xc02506cb40, 0xc08da96ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08d298fc0}, 0x660089, {0xc065cddd40, {0x174301104edebd10, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021daa000, {0x63c85b8, 0xc08d298fc0}, 0xc07268f200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104edebd10, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104edebd10, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08d298f30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d298f30}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08d298f30}, {0xc00102e4e0}, 0xc07268f100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08d298ed0}, 0xc07268f100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc08d298ba0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810798 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc041fc44b0}, 0xc063bc6a38, 0x1, 0xc01fc270a8, 0x519bbc, 0xc01a04a7e0, 0xc09e284d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc041fc44b0}, 0x1e0, 0xc01fc27230, 0x1a04a780, 0x0, 0x660069, 0xc01a04a7e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc041fc44b0}, 0xc01a04a780, {{{{0x0, 0x0}, {0xc07a96d440, 0x102}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc041fc44b0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc041fc44b0}, {0xc0171918c0, {0x17430110717e616f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc041fc44b0}, 0xc051713c20, 0xc01fc279f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041fc44b0}, 0x660089, {0xc0171918c0, {0x17430110717e616f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc041fc44b0}, 0xc05696cc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110717e616f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110717e616f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041fc4420}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041fc4420}, {0xc01fc293f8, 0x660089}, 0xc01fc29400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041fc4420}, {0xc0015221e0}, 0xc05696c700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041fc4390}, 0xc05696c700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}, 0xc00833cae0, 0xc01f447638) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}, 0x3, 0xc0436f2540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fc29980}}}, {0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}, 0xc00833cae0, 0xc0436f2540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041fc4390}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041fc4390}, {0x40, 0xc01fc29b70}, 0xc01fc29ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041fc4390}, {0x4e9d7a0, 0xc05696c700}, 0xc00833cae0, 0xc01f447638) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041fc4390}, 0xc014056540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc017191680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc017191680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809149 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc03f704cf0}, 0xc039707b48, 0x1, 0xc015bd90a8, 0x519bbc, 0xc050f9f320, 0xc0758a7140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03f704cf0}, 0x1e0, 0xc015bd9230, 0x50f9f2c0, 0x0, 0x660069, 0xc050f9f320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc03f704cf0}, 0xc050f9f2c0, {{{{0x0, 0x0}, {0xc0874338c0, 0x5a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03f704cf0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc03f704cf0}, {0xc07adf9200, {0x174301106346bdcd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc03f704cf0}, 0xc01858b400, 0xc015bd99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03f704cf0}, 0x660089, {0xc07adf9200, {0x174301106346bdcd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03f704cf0}, 0xc068918200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106346bdcd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106346bdcd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03f704c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f704c00}, {0xc015bdb3f8, 0x660089}, 0xc015bdb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03f704c00}, {0x7fe4292d0468}, 0xc043615f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03f704b70}, 0xc043615f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}, 0xc03c1d1640, 0xc048ff8210) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}, 0x3, 0xc06c402980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc015bdb980}}}, {0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}, 0xc03c1d1640, 0xc06c402980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03f704b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f704b70}, {0x40, 0xc015bdbb70}, 0xc015bdbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03f704b70}, {0x4e9d7a0, 0xc043615f00}, 0xc03c1d1640, 0xc048ff8210) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03f704b70}, 0xc0607bf500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07adf8fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07adf8fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807317 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc082c82c60}, 0xc060ee9f08, 0x1, 0xc0040a70a8, 0x519bbc, 0xc089f4ba40, 0xc06ec42f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc082c82c60}, 0x120, 0xc0040a7230, 0x89f4b9e0, 0x0, 0x660069, 0xc089f4ba40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc082c82c60}, 0xc089f4b9e0, {{{{0x0, 0x0}, {0xc0d08ae7e0, 0xa}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc082c82c60}, 0x6352230, 0x5591d488) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc082c82c60}, {0xc089f4b8c0, {0x1743011057457e18, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc082c82c60}, 0xc0463c0f00, 0xc0040a79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc082c82c60}, 0x660089, {0xc089f4b8c0, {0x1743011057457e18, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc082c82c60}, 0xc097e77000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057457e18, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057457e18, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc082c82bd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082c82bd0}, {0xc0040a93f8, 0x660089}, 0xc0040a9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc082c82bd0}, {0x0}, 0xc097e76e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc082c82b40}, 0xc097e76e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}, 0xc06b9efc00, 0xc064c6a990) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}, 0x3, 0xc05851e940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0040a9980}}}, {0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}, 0xc06b9efc00, 0xc05851e940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc082c82b40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082c82b40}, {0x40, 0xc0040a9b70}, 0xc0040a9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc082c82b40}, {0x4e9d7a0, 0xc097e76e00}, 0xc06b9efc00, 0xc064c6a990) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc082c82b40}, 0xc066a44480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc089f4b680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc089f4b680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812652 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812572 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809999 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc06dd62d20}, 0xc0411c8498, 0x1, 0xc01a85d0a8, 0x519bbc, 0xc033a4ad80, 0xc0ad668420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06dd62d20}, 0x1e0, 0xc01a85d230, 0x33a4ad20, 0x0, 0x660069, 0xc033a4ad80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc06dd62d20}, 0xc033a4ad20, {{{{0x0, 0x0}, {0xc05112d440, 0xf5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06dd62d20}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc06dd62d20}, {0xc09d32cd80, {0x17430110637a4fbc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc06dd62d20}, 0xc0637dac80, 0xc01a85d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06dd62d20}, 0x660089, {0xc09d32cd80, {0x17430110637a4fbc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc06dd62d20}, 0xc0b31c1100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110637a4fbc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110637a4fbc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06dd62c90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dd62c90}, {0xc01a85f3f8, 0x660089}, 0xc01a85f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06dd62c90}, {0x7fe429fe5b90}, 0xc0b31c0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06dd62c00}, 0xc0b31c0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}, 0xc01ff34f00, 0xc038bcf818) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}, 0x3, 0xc0c8977fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a85f980}}}, {0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}, 0xc01ff34f00, 0xc0c8977fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06dd62c00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dd62c00}, {0x40, 0xc01a85fb70}, 0xc01a85fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06dd62c00}, {0x4e9d7a0, 0xc0b31c0f00}, 0xc01ff34f00, 0xc038bcf818) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06dd62c00}, 0xc018d12780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc09d32cc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc09d32cc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812569 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219130979 [sync.Cond.Wait, 1 minutes]: sync.runtime_notifyListWait(0xc07059f3d0, 0xa) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc087bbea80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc087bbf0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06d7f6600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06d7f6600, {0x63c8510, 0xc07059f400}, 0xc06b41ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0628e9820) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc07059f400}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0628e9820) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812771 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812664 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812670 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219150674 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0b00d4d00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1131 +0x245 created by google.golang.org/grpc/internal/transport.NewServerTransport google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:341 +0x19d7 goroutine 219811776 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e78ac80, {0x63c85b8, 0xc04d7f75f0}, 0xc0208f1cc8, 0x1, 0x708018187de46c, 0x37e11d600, 0xc029488060, 0xc05281f4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04d7f75f0}, 0x120, 0xc02f993230, 0x29488000, 0x0, 0x660069, 0xc029488060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e78ac80, {0x63c85b8, 0xc04d7f75f0}, 0xc029488000, {{{{0x0, 0x0}, {0xc0c4e057a0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d7f75f0}, 0x6352410, 0x11fab538) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0180e3bd0, {0x63c85b8, 0xc04d7f75f0}, {0xc040809e60, {0x174301107adc4512, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0180e3bd0, {0x63c85b8, 0xc04d7f75f0}, 0xc06b7bc640, 0xc02f9939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d7f75f0}, 0x660089, {0xc040809e60, {0x174301107adc4512, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e7d3000, {0x63c85b8, 0xc04d7f75f0}, 0xc04a724800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107adc4512, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107adc4512, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d7f7560}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d7f7560}, {0xc02f9953f8, 0x660089}, 0xc02f995400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d7f7560}, {0x7fe326347558}, 0xc04a724500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d7f74a0}, 0xc04a724500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}, 0xc029ae9860, 0xc0208f1c98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}, 0x3, 0xc05aca1a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02f995980}}}, {0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}, 0xc029ae9860, 0xc05aca1a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d7f74a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d7f74a0}, {0x40, 0xc02f995b70}, 0xc02f995ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d7f74a0}, {0x4e9d7a0, 0xc04a724500}, 0xc029ae9860, 0xc0208f1c98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d7f74a0}, 0xc025313ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc040809d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc040809d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811098 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc0289eeff0}, 0xc03650b638, 0x1, 0xc03206b0a8, 0x519bbc, 0xc074972660, 0xc06e73a420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0d2e69c20, {0x63c85b8, 0xc0289eeff0}, 0xc0d2e69cb0, 0xc03206b1a8, 0x40, 0x60, 0xffffffffffffffff, 0xc074972660, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc0289eeff0}, 0xc0749725a0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04bc278c0, 0x1c}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0289eeff0}, 0x6352260, 0x3a7e11e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc0289eeff0}, {0xc074972480, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc0289eeff0}, 0xc07cc6e320, 0xc03206b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0289eeff0}, 0x660089, {0xc074972480, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc0289eeff0}, 0xc067fa9800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0289eef60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0289eef60}, {0xc03206d3f8, 0x660089}, 0xc03206d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0289eef60}, {0xc01c6b8400}, 0xc067fa9600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0289eee70}, 0xc067fa9600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}, 0xc093038fe0, 0xc0cd98d608) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}, 0x3, 0xc032988dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03206d980}}}, {0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}, 0xc093038fe0, 0xc032988dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0289eee70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0289eee70}, {0x40, 0xc03206db70}, 0xc03206dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0289eee70}, {0x4e9d7a0, 0xc067fa9600}, 0xc093038fe0, 0xc0cd98d608) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0289eee70}, 0xc03e80d3e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc074972360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc074972360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806185 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fe85c00, {0x63c85b8, 0xc04a828090}, 0xc0a6df9b00, 0xc06c8e66e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fe85c00, {0x63c85b8, 0xc04a828090}, 0xc0a6df9b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c8488ec, 0x0, 0x0}, 0xc076d73410, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c8488ec, 0x0, 0x0}, 0xc076d73410, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04a828000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a828000}, {0xc007e2b3f8, 0x660089}, 0xc007e2b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04a828000}, {0x3241382f6e6f2b75}, 0xc0a6df9700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0b00f7f50}, 0xc0a6df9700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}, 0xc0582dbea0, 0xc08a455050) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}, 0x3, 0xc068b49440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007e2b980}}}, {0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}, 0xc0582dbea0, 0xc068b49440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0b00f7f50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b00f7f50}, {0x40, 0xc007e2bb70}, 0xc007e2bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0b00f7f50}, {0x4e9d7a0, 0xc0a6df9700}, 0xc0582dbea0, 0xc08a455050) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0b00f7f50}, 0xc039453aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03a66efc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03a66efc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 210818698 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3dfe60, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc048e75d00, 0xc03404b000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc048e75d00, {0xc03404b000, 0x152f, 0x152f}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc048e75d00, {0xc03404b000, 0x152f, 0xc0677b9040}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc02e73cf00, {0xc03404b000, 0x140, 0xc03404b005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc023ec9560, {0xc03404b000, 0x48578c, 0x152f}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc041138438, {0xc03404b000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0687d13f8, {0x631b900, 0xc041138438}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0687d1180, {0x7fe42ab59310, 0xc023ec9560}, 0x152f) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0687d1180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0687d1180, {0xc0100bf000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc036cb1950, {0xc0100bf000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc005dc2b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc005dc2b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc005dc30e0, {0x634aac0, 0xc005dc2b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc005dc2a80, 0xc049b839a0, 0xc05ecab42b, 0xc05ecab42d, {0x635bec0, 0xc065ecbdd0}, 0xc000d6ae00, 0xc03a0bae28, 0xc05ecab429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc005dc2a80, {0x63c85b8, 0xc073e7d680}, 0xc049b83990, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0666ea021, 0x8}}, 0x0, 0xc073e7d620, 0xc073e7d650, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc073e7d680}, {0x643d810, 0xc023ec9560}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07b6ac3c0}, {0x643d810, 0xc023ec9560}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07b6ac3c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803973 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc04ba46bd0}, 0xc039a33900, 0xc03b364e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04ba46bd0}, 0xc039a33900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ba46b40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ba46b40}, {0xc0624453f8, 0x660089}, 0xc062445400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ba46b40}, {0xc04155ed00}, 0xc039a33600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ba46ab0}, 0xc039a33600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}, 0xc072246940, 0xc0266a60d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}, 0x3, 0xc09cad63c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062445980}}}, {0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}, 0xc072246940, 0xc09cad63c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ba46ab0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ba46ab0}, {0x40, 0xc062445b70}, 0xc062445ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ba46ab0}, {0x4e9d7a0, 0xc039a33600}, 0xc072246940, 0xc0266a60d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ba46ab0}, 0xc031b08720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050cff8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050cff8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219654081 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0417ff5d0, 0x19) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0789e5500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0789e5b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc013997300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc013997300, {0x63c8510, 0xc0417ff600}, 0xc054fd7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04b0194f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0417ff600}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04b0194f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219806153 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc0164a8d80}, 0xc0234eb638, 0x1, 0xc0052a70a8, 0x519bbc, 0xc0c36b89c0, 0xc073a67a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc07d47de60, {0x63c85b8, 0xc0164a8d80}, 0xc07d47df68, 0xc0052a71a8, 0x18, 0x18, 0xffffffffffffffff, 0xc0c36b89c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc0164a8d80}, 0xc0c36b8960, {{{{0xc07c654b40, 0xf}, {0xc072bf2ea0, 0x1c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x18, {0x63c85b8, 0xc0164a8d80}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc0164a8d80}, {0xc04eb8d8c0, {0x174301104c781366, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc0164a8d80}, 0xc05c5e3ae0, 0xc0052a79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0164a8d80}, 0x660089, {0xc04eb8d8c0, {0x174301104c781366, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc0164a8d80}, 0xc092639400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c781366, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c781366, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0164a8cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0164a8cf0}, {0xc0052a93f8, 0x660089}, 0xc0052a9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0164a8cf0}, {0x3d7756454c4c6c33}, 0xc092639200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0164a8c60}, 0xc092639200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}, 0xc064e67340, 0xc08df3c360) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}, 0x3, 0xc0678791c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0052a9980}}}, {0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}, 0xc064e67340, 0xc0678791c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0164a8c60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0164a8c60}, {0x40, 0xc0052a9b70}, 0xc0052a9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0164a8c60}, {0x4e9d7a0, 0xc092639200}, 0xc064e67340, 0xc08df3c360) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0164a8c60}, 0xc05b356ea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc050ff0000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc050ff0000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218906051 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020c61400, {0x63c85b8, 0xc0479faf90}, 0xc0a48c7900, 0xc06869cbe0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c61400, {0x63c85b8, 0xc0479faf90}, 0xc0a48c7900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110234b556f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110234b556f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0479faed0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0479faed0}, {0xc0820da7f8, 0x660089}, 0xc0820da800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0479faed0}, {0xc00102e4e0}, 0xc0a48c7800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0479fae70}, 0xc0a48c7800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc044326790, {0x63c85b8, 0xc0479facf0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc034b54c00, {0x63c85b8, 0xc0479facf0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc034b54c00, {0x63c8510, 0xc09a6ed900}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc034b54c00, {0x63c8510, 0xc09a6ed900}, {0x63f7db8, 0x9bf6f48}, 0xc0820df088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc034b54c00, {0x63c85b8, 0xc06ce12540}, {{0x63f7db8, 0x9bf6f48}, {0xc08f3319d3, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc034b54c00, {0x63c85b8, 0xc06ce12540}, {{0x63f7db8, 0x9bf6f48}, {0xc08f3319d3, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc08f3319d3, 0x6}, 0x0, 0x0}, {0x835442f, 0xedb7a834f, 0x0}, {0x8354919, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc034b54c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc034b54c00, {0x63c8510, 0xc016013340}, 0xc0d17c3dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc053091e90) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc016013340}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc053091e90) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802660 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f658000, {0x63c85b8, 0xc02f1179e0}, 0xc0890a9100, 0xc02225f680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f658000, {0x63c85b8, 0xc02f1179e0}, 0xc0890a9100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffd6e243f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffd6e243f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f1174d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f1174d0}, {0xc04250f3f8, 0x660089}, 0xc04250f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f1174d0}, {0x0}, 0xc0890a8f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f116d20}, 0xc0890a8f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}, 0xc039a9fb40, 0xc07db20b88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}, 0x3, 0xc05f65e880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04250f980}}}, {0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}, 0xc039a9fb40, 0xc05f65e880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f116d20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f116d20}, {0x40, 0xc04250fb70}, 0xc04250fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f116d20}, {0x4e9d7a0, 0xc0890a8f00}, 0xc039a9fb40, 0xc07db20b88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f116d20}, 0xc061977980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04b03a7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04b03a7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803823 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01e7d3000, {0x63c85b8, 0xc02d19ee40}, 0xc067528200, 0xc05eea9c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e7d3000, {0x63c85b8, 0xc02d19ee40}, 0xc067528200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110312b55e5, 0x0, 0x0}, 0xc052083c90, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110312b55e5, 0x0, 0x0}, 0xc052083c90, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02d19ed80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d19ed80}, {0xc0744d53f8, 0x660089}, 0xc0744d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02d19ed80}, {0xc0015221e0}, 0xc067528000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02d19ecf0}, 0xc067528000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}, 0xc073f7f2c0, 0xc02cc7f0f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}, 0x3, 0xc03c4cfd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0744d5980}}}, {0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}, 0xc073f7f2c0, 0xc03c4cfd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02d19ecf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d19ecf0}, {0x40, 0xc0744d5b70}, 0xc0744d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02d19ecf0}, {0x4e9d7a0, 0xc067528000}, 0xc073f7f2c0, 0xc02cc7f0f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02d19ecf0}, 0xc07d44ed80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04fd10120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04fd10120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806207 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021e97c00, {0x63c85b8, 0xc07a9b0ae0}, 0xc002358e00, 0xc05b3f6460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021e97c00, {0x63c85b8, 0xc07a9b0ae0}, 0xc002358e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d74e600, 0x0, 0x0}, 0xc07ca9d080, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d74e600, 0x0, 0x0}, 0xc07ca9d080, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a9b0a50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a9b0a50}, {0xc004e513f8, 0x660089}, 0xc004e51400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a9b0a50}, {0x7fe4287c3510}, 0xc036885e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a9b09c0}, 0xc036885e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}, 0xc0859cc160, 0xc08ea9f6e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}, 0x3, 0xc07670ae40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004e51980}}}, {0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}, 0xc0859cc160, 0xc07670ae40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a9b09c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a9b09c0}, {0x40, 0xc004e51b70}, 0xc004e51ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a9b09c0}, {0x4e9d7a0, 0xc036885e00}, 0xc0859cc160, 0xc08ea9f6e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a9b09c0}, 0xc06dd9eb40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc043d79320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc043d79320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808305 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc07741f5f0}, 0xc041f28300, 0x1, 0xc0052270a8, 0x519bbc, 0xc05bd495c0, 0xc059e98960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07741f5f0}, 0x120, 0xc005227230, 0x5bd49560, 0x0, 0x660069, 0xc05bd495c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc07741f5f0}, 0xc05bd49560, {{{{0x0, 0x0}, {0xc061a91200, 0x40}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07741f5f0}, 0x6352410, 0x43390490) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc07741f5f0}, {0xc05bd49440, {0x174301105bddeebb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc07741f5f0}, 0xc05882e460, 0xc0052279f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07741f5f0}, 0x660089, {0xc05bd49440, {0x174301105bddeebb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc07741f5f0}, 0xc059a06600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105bddeebb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105bddeebb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07741f560}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07741f560}, {0xc0052293f8, 0x660089}, 0xc005229400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07741f560}, {0xc0015221e0}, 0xc059a06400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07741f4d0}, 0xc059a06400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}, 0xc035706020, 0xc020a1b128) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}, 0x3, 0xc032843300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005229980}}}, {0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}, 0xc035706020, 0xc032843300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07741f4d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07741f4d0}, {0x40, 0xc005229b08}, 0xc005229ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07741f4d0}, {0x4e9d7a0, 0xc059a06400}, 0xc035706020, 0xc020a1b128) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07741f4d0}, 0xc08b580420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05bd49320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05bd49320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219685141 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1e90, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06ae7e800, 0xc08dbe0000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06ae7e800, {0xc08dbe0000, 0x2b7d, 0x2b7d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06ae7e800, {0xc08dbe0000, 0x2b7d, 0xc0452b8c80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc057085920, {0xc08dbe0000, 0x7eda79, 0xc08dbe0005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc047ac3ce0, {0xc08dbe0000, 0x0, 0x2b7d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc028de19f8, {0xc08dbe0000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0511fa278, {0x631b900, 0xc028de19f8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0511fa000, {0x7fe42ab59310, 0xc047ac3ce0}, 0x2b7d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0511fa000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0511fa000, {0xc09a9f3000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03a367c38, {0xc09a9f3000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc051472088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc051472088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc051472660, {0x634aac0, 0xc051472088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc051472000, 0xc0580d97c0, 0xc0831fb42b, 0xc0831fb42d, {0x635bec0, 0xc042c8b170}, 0xc000d6ae00, 0xc042f506a8, 0xc0831fb429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc051472000, {0x63c85b8, 0xc04c9099e0}, 0xc0580d97b0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc05f14d009, 0x8}}, 0x0, 0xc04c909980, 0xc04c9099b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04c9099e0}, {0x643d810, 0xc047ac3ce0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02b5cd110}, {0x643d810, 0xc047ac3ce0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02b5cd110}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805137 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b7d0000, {0x63c85b8, 0xc03d626540}, 0xc0227389f0, 0x1, 0x708017db69af4a, 0x37e11d600, 0xc056edb8c0, 0xc03bcf0540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x4677c5, {0x63c85b8, 0xc03d626540}, 0xc056edb920, 0xc07b9db1a8, 0x60, 0x60, 0x2, 0xc056edb8c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b7d0000, {0x63c85b8, 0xc03d626540}, 0xc056edb860, {{{{0x0, 0x0}, {0xc05ab56fc0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x60, {0x63c85b8, 0xc03d626540}, 0x0, 0x5) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b6f5f40, {0x63c85b8, 0xc03d626540}, {0xc072d51d40, {0x174301103e6db47f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b6f5f40, {0x63c85b8, 0xc03d626540}, 0xc02e5fa820, 0xc07b9db9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03d626540}, 0x660089, {0xc072d51d40, {0x174301103e6db47f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b7be800, {0x63c85b8, 0xc03d626540}, 0xc063d41c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103e6db47f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103e6db47f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03d626450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d626450}, {0xc07b9dd3f8, 0x660089}, 0xc07b9dd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03d626450}, {0x4d939c}, 0xc063d41a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03d6263c0}, 0xc063d41a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}, 0xc057719600, 0xc0227389c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}, 0x3, 0xc014386b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b9dd980}}}, {0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}, 0xc057719600, 0xc014386b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03d6263c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d6263c0}, {0x40, 0xc07b9ddb70}, 0xc07b9ddab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}, 0x203005, 0x203005) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03d6263c0}, {0x4e9d7a0, 0xc063d41a00}, 0xc057719600, 0xc0227389c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03d6263c0}, 0xc05501df20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc072d51b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc072d51b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805472 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff68280, {0x63c85b8, 0xc03bc93770}, 0xc04c598be8, 0x1, 0xc0044c5ff0, 0x519bbc, 0xc067337da0, 0xc0932f8960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03bc93770}, 0x120, 0xc0044c6178, 0x67337d40, 0x0, 0x660069, 0xc067337da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff68280, {0x63c85b8, 0xc03bc93770}, 0xc067337d40, {{{{0x0, 0x0}, {0xc00d3ebd40, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03bc93770}, 0x6352410, 0x3a07eb18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea3e00, {0x63c85b8, 0xc03bc93770}, {0xc067337c20, {0x17430110499c7056, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea3e00, {0x63c85b8, 0xc03bc93770}, 0xc05043d540, 0xc04c2f2938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03bc93770}, 0x660089, {0xc067337c20, {0x17430110499c7056, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff72000, {0x63c85b8, 0xc03bc93770}, 0xc0442efd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110499c7056, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110499c7056, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03bc936e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bc936e0}, {0xc04c2f4340, 0x660089}, 0xc04c2f4348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03bc936e0}, {0xc00102e4e0}, 0xc0442efc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03bc93680}, 0xc0442efc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc03bc92fc0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219143269 [select]: google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc019e65e00) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1572 +0x169 created by google.golang.org/grpc/internal/transport.newHTTP2Client google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:360 +0x1905 goroutine 219811207 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc0350e22a0}, 0xc0269c5788, 0x1, 0x0, 0x1, 0xc01b0b8180, 0xc033cf4060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0350e22a0}, 0x120, 0xc00fc58728, 0x1b0b8120, 0x0, 0x660069, 0xc01b0b8180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc0350e22a0}, 0xc01b0b8120, {{{{0x0, 0x0}, {0xc0b70a2d80, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0350e22a0}, 0x6352410, 0x50b35ff0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc0350e22a0}, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc0350e22a0}, 0xc0afd0e6e0, 0xc088136ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0350e22a0}, 0x660089, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc0350e22a0}, 0xc06df82e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0350e2210}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0350e2210}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0350e2210}, {0xc00102e4e0}, 0xc06df82d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0350e2180}, 0xc06df82d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc044fe6a50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219751114 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af29618, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0612c3180, 0xc0753fb000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0612c3180, {0xc0753fb000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0612c3180, {0xc0753fb000, 0x735, 0xc08a7f4640}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0355e6570, {0xc0753fb000, 0x7eda79, 0xc0753fb005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc03bb3e9c0, {0xc0753fb000, 0xc07bd6ec60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc049760738, {0xc0753fb000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0d0e1baf8, {0x631b900, 0xc049760738}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0d0e1b880, {0x7fe42ab59310, 0xc03bb3e9c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0d0e1b880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0d0e1b880, {0xc032f1c000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04911aae0, {0xc032f1c000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0840c5588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0840c5588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0840c5b60, {0x634aac0, 0xc0840c5588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0840c5500, 0xc03c7a69a0, 0xc07bd6f42b, 0xc07bd6f42d, {0x635bec0, 0xc05ffa02d0}, 0xc000d6ae00, 0xc083b59db8, 0xc07bd6f429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0840c5500, {0x63c85b8, 0xc06c6e2780}, 0xc03c7a6990, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc046b64037, 0x8}}, 0x0, 0xc06c6e2720, 0xc06c6e2750, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06c6e2780}, {0x643d810, 0xc03bb3e9c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02b816060}, {0x643d810, 0xc03bb3e9c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02b816060}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801635 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0728f8cf0}, 0xc026b95f20, 0x1, 0xc03eea90a8, 0x519bbc, 0xc035c10420, 0xc033a5d140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0728f8cf0}, 0x1e0, 0xc03eea9230, 0x35c103c0, 0x0, 0x660069, 0xc035c10420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0728f8cf0}, 0xc035c103c0, {{{{0x0, 0x0}, {0xc0057e26c0, 0x33}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0728f8cf0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0728f8cf0}, {0xc03cacfe60, {0x17430110062800f8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0728f8cf0}, 0xc02e444aa0, 0xc03eea99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0728f8cf0}, 0x660089, {0xc03cacfe60, {0x17430110062800f8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0728f8cf0}, 0xc046a71000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110062800f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110062800f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0728f8c60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0728f8c60}, {0xc03eeab3f8, 0x660089}, 0xc03eeab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0728f8c60}, {0x7fe429ea8340}, 0xc0621b3e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0728f8bd0}, 0xc0621b3e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}, 0xc073489940, 0xc06daaecd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}, 0x3, 0xc0669bf7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03eeab980}}}, {0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}, 0xc073489940, 0xc0669bf7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0728f8bd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0728f8bd0}, {0x40, 0xc03eeabb70}, 0xc03eeabab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0728f8bd0}, {0x4e9d7a0, 0xc0621b3e00}, 0xc073489940, 0xc06daaecd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0728f8bd0}, 0xc02cec2a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03cacfd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03cacfd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805044 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc0701d12f0}, 0xc02a7aac30, 0x1, 0xc0787170a8, 0x519bbc, 0xc070651920, 0xc0245e75c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0701d12f0}, 0x120, 0xc078717230, 0x706518c0, 0xc084e10680, 0x18, 0xc070651920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc0701d12f0}, 0xc0706518c0, {{{{0x0, 0x0}, {0xc055a9b200, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0701d12f0}, 0x6352410, 0x34331d68) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc0701d12f0}, {0xc0706517a0, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc0701d12f0}, 0xc02e903900, 0xc0787179f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0701d12f0}, 0x660089, {0xc0706517a0, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0701d12f0}, 0xc063fc0f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0701d1260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0701d1260}, {0xc0787193f8, 0x660089}, 0xc078719400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0701d1260}, {0xc0015221e0}, 0xc063fc0d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0701d11a0}, 0xc063fc0d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}, 0xc065a4b140, 0xc01fb0ad68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}, 0x3, 0xc00e1e8b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078719980}}}, {0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}, 0xc065a4b140, 0xc00e1e8b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0701d11a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0701d11a0}, {0x40, 0xc078719b70}, 0xc078719ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}, 0x203003, 0x203003) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0701d11a0}, {0x4e9d7a0, 0xc063fc0d00}, 0xc065a4b140, 0xc01fb0ad68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0701d11a0}, 0xc01838e3c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808170 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc06f433140}, 0xc02abcc7c8, 0x1, 0xc00f65b0a8, 0x519bbc, 0xc041c46f60, 0xc066d18f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06f433140}, 0x1e0, 0xc00f65b230, 0x41c46f00, 0x0, 0x660069, 0xc041c46f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc06f433140}, 0xc041c46f00, {{{{0x0, 0x0}, {0xc041e0d8c0, 0x39}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06f433140}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc06f433140}, {0xc050bacd80, {0x174301105aa31fd2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc06f433140}, 0xc072c5a3c0, 0xc00f65b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f433140}, 0x660089, {0xc050bacd80, {0x174301105aa31fd2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc06f433140}, 0xc084bb4f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aa31fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aa31fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f4330b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f4330b0}, {0xc00f65d3f8, 0x660089}, 0xc00f65d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f4330b0}, {0x0}, 0xc084bb4d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f433020}, 0xc084bb4d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}, 0xc06d02fe20, 0xc025805218) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}, 0x3, 0xc0131b0600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f65d980}}}, {0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}, 0xc06d02fe20, 0xc0131b0600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f433020}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f433020}, {0x40, 0xc00f65db70}, 0xc00f65dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f433020}, {0x4e9d7a0, 0xc084bb4d00}, 0xc06d02fe20, 0xc025805218) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f433020}, 0xc03e535c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc050bacc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc050bacc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803877 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030128800, {0x63c85b8, 0xc059985ad0}, 0xc0651b4d00, 0xc073bba640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030128800, {0x63c85b8, 0xc059985ad0}, 0xc0651b4d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011030502775, 0x0, 0x0}, 0xc083b58450, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011030502775, 0x0, 0x0}, 0xc083b58450, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059985a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059985a40}, {0xc076027a08, 0x660089}, 0xc076027a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059985a40}, {0xc00102e4e0}, 0xc0651b4c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0599859b0}, 0xc0651b4c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc059985920}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc059985920}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801770 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc09533a548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc09533a5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0929b2cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc052470a80, {0xc023d15d80, 0x2b, 0x74}, {0xc042be4de0, 0x2b, 0x2b}, 0x1, 0x2c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0929b2b88, {0xc023d15d80, 0x1, 0x2b}, {0xc042be4de0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0929b2b88, {0xc023d15d80, 0x2b, 0x74}, {0xc042be4de0, 0x2b, 0x2b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0929b22c0, {0xc042be4de0, 0x2b, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0929b22c0, {0xc023d15d80, 0x2b, 0xc042be4de0}, {0xc042be4de0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0929b2000, {0xc042be4de0, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc055f57188, {{0xc03500ce40, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc03de43700, {{0xc03500ce40, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc044b50800, {0x63c85b8, 0xc027462000}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc027462000}, {0x64a6560, 0xc03de43700}, {0xc03500ce40, 0x74636166756e616d, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc027462000}, {0x6466ed8, 0xc055f57180}, {0xc03500ce40, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc071e6f2f0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06bec8200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020e12c00, {0x63c85b8, 0xc027462000}, 0xc06bec8200, 0xc0499d6000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020e12c00, {0x63c85b8, 0xc027462000}, 0xc06bec8200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020aa7c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020aa7c50}, {0xc0076c88f0, 0x660089}, 0xc0076c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020aa7c50}, {0xc00102e4e0}, 0xc06bec8100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020aa7bf0}, 0xc06bec8100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618cae10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802687 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc045df7050}, 0xc0b9ef0800, 0xc039c89900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc045df7050}, 0xc0b9ef0800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102483d1de, 0x0, 0x0}, 0xc03c6a6c80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102483d1de, 0x0, 0x0}, 0xc03c6a6c80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045df6fc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045df6fc0}, {0xc0554233f8, 0x660089}, 0xc055423400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045df6fc0}, {0x7fe3ba7da478}, 0xc0b9ef0500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045df6f30}, 0xc0b9ef0500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}, 0xc03d2536a0, 0xc0409e8030) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}, 0x3, 0xc020497780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055423980}}}, {0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}, 0xc03d2536a0, 0xc020497780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045df6f30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045df6f30}, {0x40, 0xc055423b70}, 0xc055423ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045df6f30}, {0x4e9d7a0, 0xc0b9ef0500}, 0xc03d2536a0, 0xc0409e8030) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045df6f30}, 0xc02ea261e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219797199 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc042ea90e0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc041e1ee70, {0x4d83cc0, 0xc0b21c7880}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x11e) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc042ea8d80, 0xc090a56fa8, 0xc090a56f78) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc042ea8d80, {0x4d83cc0, 0xc0b21c7880}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc020dae0f0}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc095daeb00}, {0x4d83cc0, 0xc0b21c7880}, 0x2, {0xc00c5825c0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc020dae0f0}, {0x4f6406d, 0x7fe457d80d28}, {0x4e9d7a0, 0xc095daeb00}, {0x4d83cc0, 0xc0b21c7880}, 0x3ef7e720e74dcf85, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x392ae46ccb7878e1, {0x63c85b8, 0xc020dae0f0}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc095daeb00}, {0x4d83cc0, 0xc0b21c7880}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0655a27d8, {0x63c85b8, 0xc020dae0f0}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc0655a27d8}}, {0x63c85b8, 0xc020dae0f0}, 0xc095daeb00, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1b, 0x1a, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc04a9c6000, {0x63c85b8, 0xc020dae090}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc04a9c6000, {0x63c85b8, 0xc020dae090}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc03ebbb810, {0x63c85b8, 0xc020dae090}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0cb22d180, {0x63c85b8, 0xc020dae090}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc040603400) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0286f3280) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc03739a9c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).next(0xc05bf95080) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:374 +0x7d github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc05bf95080) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:189 +0x32c github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0286f3300) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0a92f9500) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc03fd50f20, {0x63c8510, 0xc0286f3240}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0406059f0, {0x63c8510, 0xc0286f3240}, 0xc041e1e6e0) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc0286f2b80}, 0xc09f7ceee0, 0xc041e1e6e0, 0xc0b21c7680, 0xc0876f5180, 0xc024514fe8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc0286f2b80}, 0xc024514cf0, 0xc09f7ceee0, 0xc072e0f140, {{0x63ca3e8, 0xc0b21c7600}, 0x0}, 0xc0876f5180) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc024514600, {0x63c8510, 0xc0286f2b80}, 0xc024514cf0, 0xc0286f2b80, {0x7fe42b790cf0, 0xc072e0f140}, 0x0, 0xc03739a958) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc024514600, {0x63c8510, 0xc0286f2b80}, 0xc024514cf0, {0x7fe42b790cf0, 0xc072e0f140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc024514600, {0x63c85b8, 0xc0707b4870}, {{0x63f8d78, 0xc038d7f860}, {0xc08bdc86f4, 0x70}, 0x2, 0x1}, 0xc0117ac000, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc024514600, {0x63c85b8, 0xc0707b4870}, {{0x63f8d78, 0xc038d7f860}, {0xc08bdc86f4, 0x70}, 0x2, 0x1}, 0xc0117ac000, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc024514600, {0x63c85b8, 0xc0707b4870}, {0xc0117ac000, {0xc02604bde0, 0x2, 0x2}, {0xc05621bf60, 0xe, 0xe}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc08bdc87fa, 0x0}, 0x0, {0xc3f8c48, 0xedb7a8351, 0x0}, 0x1}, 0xc024514600, 0xc090a5d980, 0xc090a5d970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc024514600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc024514600, {0x63c8510, 0xc04d17c1c0}, 0xc0768f1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0594d5e90) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc04d17c1c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0594d5e90) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812610 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc0929f3c00, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc03d68d198, 0xc079af2e38, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc079af2f50, 0x0, 0xc079af2f50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc079af2de8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc079af2de8, {0xc02e48f200, 0x18, 0x72}, {0xc061624460, 0x18, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc079af1bc0, {0xc061624460, 0x0, 0xc03d68d4e0}, 0x121ea30, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc079af1bc0, {0xc02e48f200, 0x18, 0x6d}, {0xc061624460, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc079af1900, {0xc061624460, 0x18, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0c33bd088, {{0xc08467a948, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc01b85cb00, {{0xc08467a948, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f712000, {0x63c85b8, 0xc04f63b800}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04f63b800}, {0x64a6560, 0xc01b85cb00}, {0xc08467a948, 0x465f50325011460a, 0x18}, {0x1743011094a5ab77, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04f63b800}, {0x6466ed8, 0xc0c33bd080}, {0xc08467a948, 0x17, 0x18}, {0x1743011094a5ab77, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0734d6258}, {{0x1743011094a5ab77, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011094a5ab77, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0455a3600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b2731077, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc04f63b800}, 0xc0455a3600, 0xc05ac77680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc04f63b800}, 0xc0455a3600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011094a5ab77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011094a5ab77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63b770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b770}, {0xc03d6933f8, 0x660089}, 0xc03d693400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63b770}, {0x7fe3952e7308}, 0xc0455a3400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63b6e0}, 0xc0455a3400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}, 0xc03ebdd580, 0xc031cfb0f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}, 0x3, 0xc033d33ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03d693980}}}, {0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}, 0xc03ebdd580, 0xc033d33ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63b6e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b6e0}, {0x40, 0xc03d693b70}, 0xc03d693ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63b6e0}, {0x4e9d7a0, 0xc0455a3400}, 0xc03ebdd580, 0xc031cfb0f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63b6e0}, 0xc039f0a900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e065d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e065d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804731 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc073b6dc50}, 0xc031beb980, 0x1, 0x0, 0x1, 0xc06d3b6840, 0xc04fa54f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc073b6dc50}, 0x120, 0xc08dece728, 0x6d3b67e0, 0xc084e11520, 0x18, 0xc06d3b6840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc073b6dc50}, 0xc06d3b67e0, {{{{0x0, 0x0}, {0xc048501d40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073b6dc50}, 0x6352410, 0x199f0560) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc073b6dc50}, {0xc06d3b65a0, {0x17430110310ddab1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc073b6dc50}, 0xc0a6e99ea0, 0xc064c36ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073b6dc50}, 0x660089, {0xc06d3b65a0, {0x17430110310ddab1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc073b6dc50}, 0xc041739e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073b6dbc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073b6dbc0}, {0xc002fe68f0, 0x660089}, 0xc002fe68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073b6dbc0}, {0xc00102e4e0}, 0xc041739d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073b6db60}, 0xc041739d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc073b6d980}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810352 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc086079860}, 0xc032aadd40, 0x1, 0xc028f870a8, 0x519bbc, 0xc05f404060, 0xc077490d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc086079860}, 0x1e0, 0xc028f87230, 0x5f404000, 0x0, 0x660069, 0xc05f404060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc086079860}, 0xc05f404000, {{{{0x0, 0x0}, {0xc02e6966c0, 0x90}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc086079860}, 0x63521a0, 0x43a295f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc086079860}, {0xc0483c8d80, {0x174301106d6d1d47, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc086079860}, 0xc041d423c0, 0xc028f879f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc086079860}, 0x660089, {0xc0483c8d80, {0x174301106d6d1d47, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc086079860}, 0xc058ec9900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d6d1d47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d6d1d47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0860797d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0860797d0}, {0xc028f893f8, 0x660089}, 0xc028f89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0860797d0}, {0xc0015221e0}, 0xc058ec9600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086079710}, 0xc058ec9600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}, 0xc00c7ee800, 0xc02cfc7290) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}, 0x3, 0xc06cbd5a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028f89980}}}, {0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}, 0xc00c7ee800, 0xc06cbd5a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086079710}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086079710}, {0x40, 0xc028f89b70}, 0xc028f89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086079710}, {0x4e9d7a0, 0xc058ec9600}, 0xc00c7ee800, 0xc02cfc7290) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086079710}, 0xc0603def60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0483c8b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0483c8b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809740 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024c49b80, {0x63c85b8, 0xc0c134a7e0}, 0xc0b6653830, 0x1, 0xc01cd4b0a8, 0x519bbc, 0xc02194e840, 0xc03bc42600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0c134a7e0}, 0x120, 0xc01cd4b230, 0x2194e7e0, 0x0, 0x660069, 0xc02194e840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024c49b80, {0x63c85b8, 0xc0c134a7e0}, 0xc02194e7e0, {{{{0x0, 0x0}, {0xc07ca00000, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0c134a7e0}, 0x6352410, 0x300d18e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024c054f0, {0x63c85b8, 0xc0c134a7e0}, {0xc02194e360, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024c054f0, {0x63c85b8, 0xc0c134a7e0}, 0xc06ba588c0, 0xc01cd4b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0c134a7e0}, 0x660089, {0xc02194e360, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024cab000, {0x63c85b8, 0xc0c134a7e0}, 0xc0601c8100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c134a750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c134a750}, {0xc01cd4d3f8, 0x660089}, 0xc01cd4d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c134a750}, {0x7fe347d1eb88}, 0xc01b23bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c134a6c0}, 0xc01b23bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}, 0xc0772c29c0, 0xc08871c1c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}, 0x3, 0xc03f96bbc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01cd4d980}}}, {0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}, 0xc0772c29c0, 0xc03f96bbc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0c134a6c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c134a6c0}, {0x40, 0xc01cd4db70}, 0xc01cd4dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0c134a6c0}, {0x4e9d7a0, 0xc01b23bd00}, 0xc0772c29c0, 0xc08871c1c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0c134a6c0}, 0xc0893389c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc02194e120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc02194e120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802154 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b6f6000, {0x63c85b8, 0xc0641a3c50}, 0xc049ce4df8, 0x1, 0xc01419d0a8, 0x519bbc, 0xc017fee060, 0xc05af72720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0641a3c50}, 0x120, 0xc01419d230, 0x17fee000, 0x0, 0x660069, 0xc017fee060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b6f6000, {0x63c85b8, 0xc0641a3c50}, 0xc017fee000, {{{{0x0, 0x0}, {0xc073a706c0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0641a3c50}, 0x6352410, 0x351a98d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b6f4000, {0x63c85b8, 0xc0641a3c50}, {0xc0221b5320, {0x1743010fec5acf9a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b6f4000, {0x63c85b8, 0xc0641a3c50}, 0xc066747040, 0xc01419d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0641a3c50}, 0x660089, {0xc0221b5320, {0x1743010fec5acf9a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b6e3000, {0x63c85b8, 0xc0641a3c50}, 0xc061fff300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fec5acf9a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fec5acf9a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0641a3bc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0641a3bc0}, {0xc01419f3f8, 0x660089}, 0xc01419f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0641a3bc0}, {0x7fe39113ac90}, 0xc061ffef00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0641a3b30}, 0xc061ffef00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}, 0xc04ecda940, 0xc0302f55a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}, 0x3, 0xc06e1c6240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01419f980}}}, {0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}, 0xc04ecda940, 0xc06e1c6240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0641a3b30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0641a3b30}, {0x40, 0xc01419fb70}, 0xc01419fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0641a3b30}, {0x4e9d7a0, 0xc061ffef00}, 0xc04ecda940, 0xc0302f55a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0641a3b30}, 0xc04ad08060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0221b4ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0221b4ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811209 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03c908c80, {0x63c85b8, 0xc0350e2780}, 0xc018e70798, 0x1, 0x0, 0x1, 0xc01b0b8a80, 0xc06b247a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0350e2780}, 0x120, 0xc027edc728, 0x1b0b8a20, 0x0, 0x660069, 0xc01b0b8a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03c908c80, {0x63c85b8, 0xc0350e2780}, 0xc01b0b8a20, {{{{0x0, 0x0}, {0xc073d519e0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0350e2780}, 0x6352410, 0x6847e008) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00852fe50, {0x63c85b8, 0xc0350e2780}, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00852fe50, {0x63c85b8, 0xc0350e2780}, 0xc05ac77400, 0xc088136ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0350e2780}, 0x660089, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b078c00, {0x63c85b8, 0xc0350e2780}, 0xc06df83100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0350e2660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0350e2660}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0350e2660}, {0xc00102e4e0}, 0xc06df83000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0350e25d0}, 0xc06df83000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc044fe6b10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810119 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc066400720}, {0xc0639f7a20, 0x2, 0xc01102b658}, {0x0, {0x1743010ff4c95c4b, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc066400720}, 0x1200, 0xc043ccdb00, 0x6, 0xc02bfcb8d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc066400720}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219789999 [runnable]: sync.runtime_SemacquireMutex(0xc06f7e6690, 0x8f, 0xc06f7e66a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc06f7e6538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc06f7e6560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0c3190000, {0xffffffffffffff15, 0xc06f7e66c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0c3190000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0c3190000, {0xc02ac03e80, 0x32, 0x72}, {0xc0a2297180, 0x32, 0x71}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0bd83fe98, {0xc02ac03e80, 0x32, 0x72}, {0xc0a2297180, 0x32, 0x71}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0bd83e7c0, {0xc0a2297180, 0x71, 0x72}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0bd83e7c0, {0xc02ac03e80, 0x71, 0xc0a2297180}, {0xc0a2297180, 0xc050f0cfc0, 0x31}, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0bd83e500, {0xc0a2297180, 0x32, 0x71}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06c034828, {{0xc050f0cfc0, 0x31, 0xc0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0ae80df00, {{0xc06be45f50, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc052eb3400, {0x63c85b8, 0xc06bd82390}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06bd82390}, {0x64a6560, 0xc0ae80df00}, {0xc06be45f50, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06bd82390}, {0x6466ed8, 0xc06c034580}, {0xc06be45f50, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc046ebe528}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc047ba5200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f562400, {0x63c85b8, 0xc06bd82390}, 0xc047ba5200, 0xc0659d4c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f562400, {0x63c85b8, 0xc06bd82390}, 0xc047ba5200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06bd82300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06bd82300}, {0xc0933e48f0, 0x660089}, 0xc0933e48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06bd82300}, {0xc00102e4e0}, 0xc047ba5100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06bd822a0}, 0xc047ba5100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03ad44900}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811446 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc071098120}, 0xc052b30498, 0x1, 0xc0280790a8, 0x519bbc, 0xc04ffce4e0, 0xc033cf4a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc071098120}, 0x120, 0xc028079230, 0x4ffce480, 0x0, 0x660069, 0xc04ffce4e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc071098120}, 0xc04ffce480, {{{{0x0, 0x0}, {0xc09d5026c0, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc071098120}, 0x6352410, 0x6847e970) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc071098120}, {0xc04ffce360, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc071098120}, 0xc02d2d9680, 0xc0280799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc071098120}, 0x660089, {0xc04ffce360, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc071098120}, 0xc04e296800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc071098090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071098090}, {0xc02807b3f8, 0x660089}, 0xc02807b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc071098090}, {0x7fe429f531a0}, 0xc04e296600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f7a9fb0}, 0xc04e296600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}, 0xc004e27400, 0xc02bde3ba8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}, 0x3, 0xc0739685c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02807b980}}}, {0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}, 0xc004e27400, 0xc0739685c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f7a9fb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f7a9fb0}, {0x40, 0xc02807bb70}, 0xc02807bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f7a9fb0}, {0x4e9d7a0, 0xc04e296600}, 0xc004e27400, 0xc02bde3ba8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f7a9fb0}, 0xc02f5a39e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04ffce240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04ffce240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812663 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808304 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0623ed250, {0x63c85b8, 0xc03b3a4c60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc03b3a4c60}, {0x4f67ad2, 0x6446ff8}, 0xc016473fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 216536831 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b2310, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06c70e200, 0xc00f7ff800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06c70e200, {0xc00f7ff800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06c70e200, {0xc00f7ff800, 0x735, 0xc047263b80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc049a43718, {0xc00f7ff800, 0x7eda79, 0xc00f7ff805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc07b156b40, {0xc00f7ff800, 0x49560e0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc079319a28, {0xc00f7ff800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc05797daf8, {0x631b900, 0xc079319a28}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc05797d880, {0x7fe42ab59310, 0xc07b156b40}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc05797d880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc05797d880, {0xc0aea9c000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc063ef8f18, {0xc0aea9c000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc020213588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc020213588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc020213b60, {0x634aac0, 0xc020213588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc020213500, 0xc03020f4e0, 0xc05b95d42b, 0xc05b95d42d, {0x635bec0, 0xc0550f01b0}, 0xc000d6ae00, 0xc06c7cdff8, 0xc05b95d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc020213500, {0x63c85b8, 0xc07283f260}, 0xc03020f4d0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc04714c025, 0x8}}, 0x0, 0xc07283f200, 0xc07283f230, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc07283f260}, {0x643d810, 0xc07b156b40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06fbb87b0}, {0x643d810, 0xc07b156b40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06fbb87b0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802617 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc057475ec0}, 0xc046717278, 0x1, 0xc0547c70a8, 0x519bbc, 0xc045747ec0, 0xc00de87a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc057475ec0}, 0x120, 0xc0547c7230, 0x45747e60, 0x0, 0x660069, 0xc045747ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc057475ec0}, 0xc045747e60, {{{{0x0, 0x0}, {0xc0844206c0, 0xd}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc057475ec0}, 0x6352410, 0x5ae3d270) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc057475ec0}, {0xc045747d40, {0x1743011020dd0e6c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc057475ec0}, 0xc052cf1ea0, 0xc0547c79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc057475ec0}, 0x660089, {0xc045747d40, {0x1743011020dd0e6c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc057475ec0}, 0xc041b61e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020dd0e6c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020dd0e6c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057475e30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057475e30}, {0xc0547c93f8, 0x660089}, 0xc0547c9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057475e30}, {0x0}, 0xc041b61c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057475da0}, 0xc041b61c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}, 0xc03df79ae0, 0xc0a2826a98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}, 0x3, 0xc011fe7640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0547c9980}}}, {0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}, 0xc03df79ae0, 0xc011fe7640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc057475da0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057475da0}, {0x40, 0xc0547c9b70}, 0xc0547c9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc057475da0}, {0x4e9d7a0, 0xc041b61c00}, 0xc03df79ae0, 0xc0a2826a98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc057475da0}, 0xc097a6d0e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc053256c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc053256c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811523 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b0ff540, {0x63c85b8, 0xc019d41620}, 0xc06f29c1e0, 0x1, 0xc0172a10a8, 0x519bbc, 0xc06beaf800, 0xc03da08840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc019d41620}, 0x120, 0xc0172a1230, 0x6beaf7a0, 0x0, 0x660069, 0xc06beaf800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b0ff540, {0x63c85b8, 0xc019d41620}, 0xc06beaf7a0, {{{{0x0, 0x0}, {0xc02f75b440, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc019d41620}, 0x6352410, 0x33df4560) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0faeb0, {0x63c85b8, 0xc019d41620}, {0xc06beaf680, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0faeb0, {0x63c85b8, 0xc019d41620}, 0xc041942780, 0xc0172a19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc019d41620}, 0x660089, {0xc06beaf680, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b12a400, {0x63c85b8, 0xc019d41620}, 0xc05758c700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc019d41590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019d41590}, {0xc0172a33f8, 0x660089}, 0xc0172a3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc019d41590}, {0x7fe13dddd260}, 0xc05758c500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc019d41500}, 0xc05758c500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}, 0xc019167740, 0xc025186438) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}, 0x3, 0xc00dd3c1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0172a3980}}}, {0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}, 0xc019167740, 0xc00dd3c1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc019d41500}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019d41500}, {0x40, 0xc0172a3b70}, 0xc0172a3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc019d41500}, {0x4e9d7a0, 0xc05758c500}, 0xc019167740, 0xc025186438) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc019d41500}, 0xc03148f020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06beaf560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06beaf560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807983 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc059f83ef0}, 0xc0614fd368, 0x1, 0xc006a990a8, 0x519bbc, 0xc0750c56e0, 0xc0797da2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc059f83ef0}, 0x120, 0xc006a99230, 0x750c5680, 0x0, 0x660069, 0xc0750c56e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc059f83ef0}, 0xc0750c5680, {{{{0x0, 0x0}, {0xc093240480, 0x11e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc059f83ef0}, 0x6352410, 0x66762dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc059f83ef0}, {0xc0750c5560, {0x174301105ae80c43, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc059f83ef0}, 0xc076de52c0, 0xc006a999f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc059f83ef0}, 0x660089, {0xc0750c5560, {0x174301105ae80c43, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc059f83ef0}, 0xc0452db500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105ae80c43, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105ae80c43, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059f83e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059f83e60}, {0xc006a9b3f8, 0x660089}, 0xc006a9b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059f83e60}, {0x3feffffffd645321}, 0xc0452db300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc059f83dd0}, 0xc0452db300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}, 0xc06f052420, 0xc0293d9098) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}, 0x3, 0xc0388ce140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006a9b980}}}, {0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}, 0xc06f052420, 0xc0388ce140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc059f83dd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059f83dd0}, {0x40, 0xc006a9bb70}, 0xc006a9bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc059f83dd0}, {0x4e9d7a0, 0xc0452db300}, 0xc06f052420, 0xc0293d9098) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc059f83dd0}, 0xc068968ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0750c5440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0750c5440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812643 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812673 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811536 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc06ae5c0f0}, 0xc00b73fe00, 0x1, 0xc034cc50a8, 0x519bbc, 0xc026e87da0, 0xc019f3ccc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06ae5c0f0}, 0x120, 0xc034cc5230, 0x26e87d40, 0x0, 0x660069, 0xc026e87da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc06ae5c0f0}, 0xc026e87d40, {{{{0x0, 0x0}, {0xc059c9cfc0, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06ae5c0f0}, 0x6352410, 0x47a9d6d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc06ae5c0f0}, {0xc026e87c20, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc06ae5c0f0}, 0xc039b4b540, 0xc034cc59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06ae5c0f0}, 0x660089, {0xc026e87c20, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc06ae5c0f0}, 0xc0b7b9e800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ae5c060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ae5c060}, {0xc034cc73f8, 0x660089}, 0xc034cc7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ae5c060}, {0xc0015221e0}, 0xc0b7b9e600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c7fffb0}, 0xc0b7b9e600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}, 0xc03cd3e0c0, 0xc0a7338ca8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}, 0x3, 0xc078b8b500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034cc7980}}}, {0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}, 0xc03cd3e0c0, 0xc078b8b500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c7fffb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c7fffb0}, {0x40, 0xc034cc7b70}, 0xc034cc7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c7fffb0}, {0x4e9d7a0, 0xc0b7b9e600}, 0xc03cd3e0c0, 0xc0a7338ca8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c7fffb0}, 0xc0b860e1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0522af0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0522af0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806119 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0302d2780, {0x63c85b8, 0xc0356e5650}, 0xc0662f4450, 0x1, 0xc00665d0a8, 0x519bbc, 0xc077cffce0, 0xc04d749740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc057368fc0, {0x63c85b8, 0xc0356e5650}, 0xc077cffce0, 0xc00665d1a8, 0x31, 0x40, 0x1, 0xc077cffce0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0302d2780, {0x63c85b8, 0xc0356e5650}, 0xc077cffc20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc051b778c0, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0356e5650}, 0x6352260, 0x1cdd9a68) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030161cc0, {0x63c85b8, 0xc0356e5650}, {0xc077cffb00, {0x174301100fab231d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030161cc0, {0x63c85b8, 0xc0356e5650}, 0xc06d61cf00, 0xc00665d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0356e5650}, 0x660089, {0xc077cffb00, {0x174301100fab231d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0302e3800, {0x63c85b8, 0xc0356e5650}, 0xc060bb7500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100fab231d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100fab231d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0356e55c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0356e55c0}, {0xc00665f3f8, 0x660089}, 0xc00665f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0356e55c0}, {0x0}, 0xc060bb7300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0356e5530}, 0xc060bb7300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}, 0xc073ecc0e0, 0xc086e671a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}, 0x3, 0xc058fe3100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00665f980}}}, {0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}, 0xc073ecc0e0, 0xc058fe3100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0356e5530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0356e5530}, {0x40, 0xc00665fb70}, 0xc00665fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0356e5530}, {0x4e9d7a0, 0xc060bb7300}, 0xc073ecc0e0, 0xc086e671a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0356e5530}, 0xc03b3b01e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc077cff9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc077cff9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802460 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc0c78b5ad0}, {0xc056a8e160, 0x2, 0xc09c0c7658}, {0x0, {0x1743010feec6bb36, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc0c78b5ad0}, 0x18701, 0xc0479b39e0, 0x38, 0xc0538594b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc0c78b5ad0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808748 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e284f00, {0x63c85b8, 0xc028f2cdb0}, 0xc053c6ee88, 0x1, 0xc00796b0a8, 0x519bbc, 0xc079bbd5c0, 0xc058536760) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc028f2cdb0}, 0x120, 0xc00796b230, 0x79bbd560, 0x0, 0x660069, 0xc079bbd5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e284f00, {0x63c85b8, 0xc028f2cdb0}, 0xc079bbd560, {{{{0x0, 0x0}, {0xc0735e5e60, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc028f2cdb0}, 0x6352230, 0x57084478) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e2a4500, {0x63c85b8, 0xc028f2cdb0}, {0xc079bbd440, {0x1743011058ad5e90, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e2a4500, {0x63c85b8, 0xc028f2cdb0}, 0xc039b4a6e0, 0xc00796b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc028f2cdb0}, 0x660089, {0xc079bbd440, {0x1743011058ad5e90, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e2e3000, {0x63c85b8, 0xc028f2cdb0}, 0xc043832e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058ad5e90, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058ad5e90, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc028f2cd20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028f2cd20}, {0xc00796d3f8, 0x660089}, 0xc00796d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc028f2cd20}, {0x3fefffff78658355}, 0xc043832c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc028f2cc90}, 0xc043832c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}, 0xc056b742a0, 0xc053c6fc20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}, 0x3, 0xc041bcdcc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00796d980}}}, {0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}, 0xc056b742a0, 0xc041bcdcc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc028f2cc90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028f2cc90}, {0x40, 0xc00796db70}, 0xc00796dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc028f2cc90}, {0x4e9d7a0, 0xc043832c00}, 0xc056b742a0, 0xc053c6fc20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc028f2cc90}, 0xc02cbf6f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc079bbd320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc079bbd320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808367 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc070d6cc90}, 0xc0506a6cf0, 0x1, 0xc00a70b0a8, 0x519bbc, 0xc05a74d260, 0xc050bc7b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc070d6cc90}, 0x120, 0xc00a70b230, 0x5a74d200, 0x0, 0x660069, 0xc05a74d260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc070d6cc90}, 0xc05a74d200, {{{{0x0, 0x0}, {0xc040439200, 0x15}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc070d6cc90}, 0x6352410, 0x667635a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc070d6cc90}, {0xc05a74d0e0, {0x174301105b52c013, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc070d6cc90}, 0xc0509355e0, 0xc00a70b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc070d6cc90}, 0x660089, {0xc05a74d0e0, {0x174301105b52c013, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc070d6cc90}, 0xc04b76aa00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b52c013, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b52c013, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070d6cc00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070d6cc00}, {0xc00a70d3f8, 0x660089}, 0xc00a70d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070d6cc00}, {0x0}, 0xc04b76a800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070d6cb70}, 0xc04b76a800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}, 0xc04a373200, 0xc0258f9d10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}, 0x3, 0xc04627cb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a70d980}}}, {0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}, 0xc04a373200, 0xc04627cb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070d6cb70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070d6cb70}, {0x40, 0xc00a70db70}, 0xc00a70dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070d6cb70}, {0x4e9d7a0, 0xc04b76a800}, 0xc04a373200, 0xc0258f9d10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070d6cb70}, 0xc052b8d380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05a74cfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05a74cfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219721213 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f5ac400, {0x63c85b8, 0xc01ff74f90}, 0xc06686a000, 0xc05ad9e3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f5ac400, {0x63c85b8, 0xc01ff74f90}, 0xc06686a000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110567df5e4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110567df5e4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01ff74f00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ff74f00}, {0xc0902c27f8, 0x660089}, 0xc0902c2800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01ff74f00}, {0xc00102e4e0}, 0xc0b3b9df00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01ff74e40}, 0xc0b3b9df00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc05383a6e0, {0x63c85b8, 0xc01ff74cc0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc070123300, {0x63c85b8, 0xc01ff74cc0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc070123300, {0x63c8510, 0xc0525617c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc070123300, {0x63c8510, 0xc0525617c0}, {0x63f7db8, 0x9bf6f48}, 0xc0902c7088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc070123300, {0x63c85b8, 0xc043655bc0}, {{0x63f7db8, 0x9bf6f48}, {0xc0a2c64da0, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc070123300, {0x63c85b8, 0xc043655bc0}, {{0x63f7db8, 0x9bf6f48}, {0xc0a2c64da0, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0a2c64da0, 0x6}, 0x0, 0x0}, {0x52cd217, 0xedb7a8350, 0x0}, {0x52cd997, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc070123300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc070123300, {0x63c8510, 0xc05279a780}, 0xc06d515dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05b943620) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc05279a780}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc05b943620) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219805752 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0122d5800, {0x63c85b8, 0xc083525470}, 0xc04f745300, 0xc0831fe8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0122d5800, {0x63c85b8, 0xc083525470}, 0xc04f745300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110480e29a0, 0x0, 0x0}, 0xc02816ab20, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110480e29a0, 0x0, 0x0}, 0xc02816ab20, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0835253e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0835253e0}, {0xc004e593f8, 0x660089}, 0xc004e59400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0835253e0}, {0x8e3c6058023786dc}, 0xc04f745000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc083525350}, 0xc04f745000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}, 0xc06f237ce0, 0xc026b79d10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}, 0x3, 0xc06e0fe980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004e59980}}}, {0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}, 0xc06f237ce0, 0xc06e0fe980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc083525350}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc083525350}, {0x40, 0xc004e59b70}, 0xc004e59ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc083525350}, {0x4e9d7a0, 0xc04f745000}, 0xc06f237ce0, 0xc026b79d10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc083525350}, 0xc0203ff9e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0937325a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0937325a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801868 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc03be459b0}, 0xc070738780, 0x1, 0xc060e350a8, 0x519bbc, 0xc03543faa0, 0xc036159500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03be459b0}, 0x1e0, 0xc060e35230, 0x3543fa40, 0x0, 0x660069, 0xc03543faa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc03be459b0}, 0xc03543fa40, {{{{0x0, 0x0}, {0xc070adab40, 0x32}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03be459b0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc03be459b0}, {0xc010a6b7a0, {0x174301101e0e3064, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc03be459b0}, 0xc087a1e780, 0xc060e359f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03be459b0}, 0x660089, {0xc010a6b7a0, {0x174301101e0e3064, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03be459b0}, 0xc06bec8c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101e0e3064, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101e0e3064, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03be458c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03be458c0}, {0xc060e373f8, 0x660089}, 0xc060e37400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03be458c0}, {0x0}, 0xc06bec8a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03be457a0}, 0xc06bec8a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}, 0xc01b01eb40, 0xc09a530b40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}, 0x3, 0xc03ed873c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc060e37980}}}, {0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}, 0xc01b01eb40, 0xc03ed873c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03be457a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03be457a0}, {0x40, 0xc060e37b70}, 0xc060e37ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03be457a0}, {0x4e9d7a0, 0xc06bec8a00}, 0xc01b01eb40, 0xc09a530b40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03be457a0}, 0xc07c7cea80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc010a6b680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc010a6b680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802682 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc086bba480}, 0xc04c765f00, 0xc083df4d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc086bba480}, 0xc04c765f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100157d7bc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100157d7bc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086bba3f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086bba3f0}, {0xc0501b93f8, 0x660089}, 0xc0501b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086bba3f0}, {0x0}, 0xc04c765d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086bba330}, 0xc04c765d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}, 0xc0270b5c40, 0xc042d4a618) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}, 0x3, 0xc05511a080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0501b9980}}}, {0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}, 0xc0270b5c40, 0xc05511a080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086bba330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086bba330}, {0x40, 0xc0501b9b70}, 0xc0501b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086bba330}, {0x4e9d7a0, 0xc04c765d00}, 0xc0270b5c40, 0xc042d4a618) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086bba330}, 0xc0228b9a40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652c7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652c7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811172 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc057ecc720}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808506 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03bd19180, {0x63c85b8, 0xc017720ab0}, 0xc047c08ba0, 0x1, 0xc00d6210a8, 0x519bbc, 0xc00fbd0060, 0xc06f381320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc017720ab0}, 0x1e0, 0xc00d621230, 0xfbd0000, 0x0, 0x660069, 0xc00fbd0060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03bd19180, {0x63c85b8, 0xc017720ab0}, 0xc00fbd0000, {{{{0x0, 0x0}, {0xc059633560, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc017720ab0}, 0x63521a0, 0x324316f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc067985680, {0x63c85b8, 0xc017720ab0}, {0xc061e2bb00, {0x174301105b77d610, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc067985680, {0x63c85b8, 0xc017720ab0}, 0xc03c3ef180, 0xc00d6219f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc017720ab0}, 0x660089, {0xc061e2bb00, {0x174301105b77d610, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc017720ab0}, 0xc0650aa100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0177209f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0177209f0}, {0xc00d6233f8, 0x660089}, 0xc00d623400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0177209f0}, {0x7fe394450888}, 0xc066b5bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0177207e0}, 0xc066b5bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}, 0xc0196b2540, 0xc02a4e90c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}, 0x3, 0xc066669bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00d623980}}}, {0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}, 0xc0196b2540, 0xc066669bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0177207e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0177207e0}, {0x40, 0xc00d623b70}, 0xc00d623ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0177207e0}, {0x4e9d7a0, 0xc066b5bf00}, 0xc0196b2540, 0xc02a4e90c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0177207e0}, 0xc021042000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07af68360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07af68360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219582189 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a5a5d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc057e9bf80, 0xc06573f000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc057e9bf80, {0xc06573f000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc057e9bf80, {0xc06573f000, 0x735, 0xc068096780}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01e0a2b18, {0xc06573f000, 0x7eda79, 0xc06573f005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0c8fe1620, {0xc06573f000, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc028de0d50, {0xc06573f000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc062a205f8, {0x631b900, 0xc028de0d50}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc062a20380, {0x7fe42ab59310, 0xc0c8fe1620}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc062a20380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc062a20380, {0xc0a83a7000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0688762b8, {0xc0a83a7000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0b31b7588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0b31b7588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0b31b7b60, {0x634aac0, 0xc0b31b7588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0b31b7500, 0xc03c4e4c60, 0xc082f6d42b, 0xc082f6d42d, {0x635bec0, 0xc078a2bd40}, 0xc000d6ae00, 0xc03c7f0ed8, 0xc082f6d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0b31b7500, {0x63c85b8, 0xc0297028a0}, 0xc03c4e4c50, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a0b93009, 0x8}}, 0x0, 0xc029702840, 0xc029702870, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0297028a0}, {0x643d810, 0xc0c8fe1620}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06bdb4570}, {0x643d810, 0xc0c8fe1620}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06bdb4570}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 213794539 [sync.Cond.Wait, 1 minutes]: sync.runtime_notifyListWait(0xc00f01f990, 0xac) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc09dc5e000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc09dc5e660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc070b8a000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc070b8a000, {0x63c8510, 0xc00f01f9c0}, 0xc07d0abdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc057e18610) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc00f01f9c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc057e18610) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804755 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0790216e0}, 0xc04cfa5698, 0x1, 0xc087f930a8, 0x519bbc, 0xc0504678c0, 0xc0513aed80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0790216e0}, 0x1e0, 0xc087f93230, 0x50467860, 0xc084e17380, 0x18, 0xc0504678c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0790216e0}, 0xc050467860, {{{{0x0, 0x0}, {0xc053a14000, 0xf4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0790216e0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0790216e0}, {0xc06623e6c0, {0x1743011038390901, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0790216e0}, 0xc050b1abe0, 0xc087f939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0790216e0}, 0x660089, {0xc06623e6c0, {0x1743011038390901, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0790216e0}, 0xc051093300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011038390901, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011038390901, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079021650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079021650}, {0xc087f953f8, 0x660089}, 0xc087f95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079021650}, {0x0}, 0xc051093100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0790215c0}, 0xc051093100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}, 0xc049344fa0, 0xc0205de7f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}, 0x3, 0xc06f0dfa00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc087f95980}}}, {0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}, 0xc049344fa0, 0xc06f0dfa00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0790215c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0790215c0}, {0x40, 0xc087f95b70}, 0xc087f95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0790215c0}, {0x4e9d7a0, 0xc051093100}, 0xc049344fa0, 0xc0205de7f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0790215c0}, 0xc01ce9a9c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06623e5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06623e5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812468 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6140, {0x63c85b8, 0xc072bf11a0}, 0xc041854828, 0x1, 0xc03bffd6d8, 0x519bbc, 0xc082445aa0, 0xc05a2680c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0476b6a20, {0x63c85b8, 0xc072bf11a0}, 0xc0476b6a58, 0xc03bffd7d8, 0x41, 0x60, 0xffffffffffffffff, 0xc082445aa0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6140, {0x63c85b8, 0xc072bf11a0}, 0xc0824459e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc065c6bc20, 0x9}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc072bf11a0}, 0x6352260, 0x33a92d78) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddbd0, {0x63c85b8, 0xc072bf11a0}, {0xc0824458c0, {0x174301105c9acd56, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddbd0, {0x63c85b8, 0xc072bf11a0}, 0xc06839f680, 0xc03bffe020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc072bf11a0}, 0x660089, {0xc0824458c0, {0x174301105c9acd56, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099c00, {0x63c85b8, 0xc072bf11a0}, 0xc01a4e4a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c9acd56, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c9acd56, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072bf10e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072bf10e0}, {0xc07cdf5a28, 0x660089}, 0xc07cdf5a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072bf10e0}, {0xc00102e4e0}, 0xc01a4e4400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072bf1080}, 0xc01a4e4400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc073dd0750, {0x63c85b8, 0xc072bf0e70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc073dd0750, {0x63c85b8, 0xc072bf0e70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc072bf0e70}, {0x4f67ad2, 0x6446ff8}, 0xc019549fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219810847 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0452dd320}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809727 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0039f7a10}, 0xc04f13e600, 0x1, 0xc0124c76d8, 0x519bbc, 0xc0906b8540, 0xc03bc001e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc078287800, {0x63c85b8, 0xc0039f7a10}, 0xc06b8470e0, 0xc0124c77d8, 0x906b8540, 0x0, 0x2, 0xc0906b8540, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0039f7a10}, 0xc0906b8480, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc028414240, 0x8d}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0039f7a10}, 0x6352260, 0x3f8eb6f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0039f7a10}, {0xc0906b8240, {0x174301102ce103ca, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0039f7a10}, 0xc0497f85a0, 0xc0124c8020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0039f7a10}, 0x660089, {0xc0906b8240, {0x174301102ce103ca, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0039f7a10}, 0xc07376fd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ce103ca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ce103ca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0039f7980}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0039f7980}, {0xc088693a28, 0x660089}, 0xc088693a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0039f7980}, {0xc00102e4e0}, 0xc07376fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0039f7920}, 0xc07376fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc051ac6cd0, {0x63c85b8, 0xc0039f7890}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc051ac6cd0, {0x63c85b8, 0xc0039f7890}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0039f7890}, {0x4f67ad2, 0x6446ff8}, 0xc088191fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219150684 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc033b526c0, 0xc00a7da980, {0x643dbd8, 0xc011ac8840}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc053f5aba0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805094 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc0592a0b40}, 0xc05b94e2e8, 0x1, 0x0, 0x1, 0xc07124f260, 0xc06edc7da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0592a0b40}, 0x120, 0xc07abe6728, 0x7124f200, 0xc084e3c000, 0x18, 0xc07124f260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc0592a0b40}, 0xc07124f200, {{{{0x0, 0x0}, {0xc0557bc900, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0592a0b40}, 0x6352410, 0x71c3e930) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc0592a0b40}, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc0592a0b40}, 0xc08f98b2c0, 0xc076d8cee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0592a0b40}, 0x660089, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc0592a0b40}, 0xc06173b000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592a0ab0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592a0ab0}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592a0ab0}, {0xc00102e4e0}, 0xc06173af00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592a0a50}, 0xc06173af00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fcdba40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219784872 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc0be91b500, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc08b7cb138, 0xc07d264938, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc07d264a50, 0x0, 0xc07d264a50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc07d2648e8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d2648e8, {0xc035007540, 0x18, 0x93}, {0xc028fe7450, 0x18, 0x48}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d2636c0, {0xc028fe7450, 0x0, 0xc08b7cb480}, 0x121ea30, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d2636c0, {0xc035007540, 0x18, 0x48}, {0xc028fe7450, 0x17, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d263400, {0xc028fe7450, 0x18, 0x48}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0564e7608, {{0xc041a01d10, 0x17, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07c618400, {{0xc041a01d10, 0x17, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0562b1400, {0x63c85b8, 0xc0288fcfc0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0288fcfc0}, {0x64a6560, 0xc07c618400}, {0xc041a01d10, 0x0, 0x30}, {0x174301109ec20d34, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0288fcfc0}, {0x6466ed8, 0xc0564e7600}, {0xc041a01d10, 0x17, 0x30}, {0x174301109ec20d34, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc078ff6768}, {{0x174301109ec20d34, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109ec20d34, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0a085eb00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bc8f7234, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc0288fcfc0}, 0xc0a085eb00, 0xc052ea66e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc0288fcfc0}, 0xc0a085eb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109ec20d34, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109ec20d34, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0288fcf30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0288fcf30}, {0xc08b7d1398, 0x660089}, 0xc08b7d13a0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0288fcf30}, {0xc00102e4e0}, 0xc0a085ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0288fced0}, 0xc0a085ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc04b6e5860, {0x63c85b8, 0xc0288fcdb0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc04b6e5860, {0x63c85b8, 0xc0288fcdb0}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc058298e10, {0x63c85b8, 0xc0288fcdb0}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0537cdc00, {0x63c85b8, 0xc0288fcdb0}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0675ae460) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0650ccec0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc0288fcb70) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/is_null_ops.eg.go:226 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc038d62a20) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0c515ac00) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc0a8558e70, {0x63c8510, 0xc0650cce00}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc044b41360, {0x63c8510, 0xc0650cce00}, 0xc073b57290) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc0650ccb00}, 0xc067fa5ea0, 0xc073b57290, 0xc0645b8a00, 0xc074f86e00, 0xc05786f5e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc0650ccb00}, 0xc05786f2f0, 0xc067fa5ea0, 0xc062bb86c0, {{0x63ca960, 0xc029f6fe00}, 0x0}, 0xc074f86e00) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc05786ec00, {0x63c8510, 0xc0650ccb00}, 0xc05786f2f0, 0xc0650ccb00, {0x7fe42b790cf0, 0xc062bb86c0}, 0x0, 0xc038d627d8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc05786ec00, {0x63c8510, 0xc0650ccb00}, 0xc05786f2f0, {0x7fe42b790cf0, 0xc062bb86c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc05786ec00, {0x63c85b8, 0xc0288fc090}, {{0x63f8d78, 0xc08dda5db0}, {0xc085438012, 0xd7}, 0x1, 0x1}, 0xc044c76120, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc05786ec00, {0x63c85b8, 0xc0288fc090}, {{0x63f8d78, 0xc08dda5db0}, {0xc085438012, 0xd7}, 0x1, 0x1}, 0xc044c76120, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc05786ec00, {0x63c85b8, 0xc0288fc090}, {0xc044c76120, {0xc04b28d830, 0x1, 0x1}, {0xc03edba920, 0x6, 0x6}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc085438150, 0x0}, 0x0, {0xc4fefa7, 0xedb7a8351, 0x0}, 0x1}, 0xc05786ec00, 0xc08b7d7980, 0xc08b7d7970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05786ec00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05786ec00, {0x63c8510, 0xc082bec100}, 0xc076195dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07d45a8c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc082bec100}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07d45a8c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804649 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4e2b8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0619de280, 0xc052d2d800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0619de280, {0xc052d2d800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0619de280, {0xc052d2d800, 0x735, 0xc04e2b9cc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0473ddcf0, {0xc052d2d800, 0x7eda79, 0xc052d2d805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01ee507e0, {0xc052d2d800, 0x4a7cec0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03c5458f0, {0xc052d2d800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06e4c7e78, {0x631b900, 0xc03c5458f0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06e4c7c00, {0x7fe42ab59310, 0xc01ee507e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06e4c7c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06e4c7c00, {0xc0da1b4000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc024690e70, {0xc0da1b4000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc053a62088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc053a62088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc053a62660, {0x634aac0, 0xc053a62088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc053a62000, 0xc04a4953b0, 0xc094b9742b, 0xc094b9742d, {0x635bec0, 0xc01250d3b0}, 0xc000d6ae00, 0xc066e97a68, 0xc094b97429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc053a62000, {0x63c85b8, 0xc0830d4510}, 0xc04a4953a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc078b56009, 0x8}}, 0x0, 0xc0830d4480, 0xc0830d44e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0830d4510}, {0x643d810, 0xc01ee507e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc044d6a3c0}, {0x643d810, 0xc01ee507e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc044d6a3c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802261 [semacquire]: sync.runtime_SemacquireMutex(0xc061af81a0, 0x3, 0xc061af81a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc0994cc338) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fe3117edbc0, 0x7f4b, 0x7fe19919f0c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc0242403a0, 0x7fe3117edb01, 0x7fe3117edbe0, 0x4388, 0x438d) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc05383e000, {0x1f19324, 0x4388}, 0x0, 0xc048319d78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc048319b00, {0xffffffffffffff15, 0xc0994cc5c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc048319b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc048319b00, {0xc073186380, 0x2b, 0x67}, 0x13, 0xf2) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc048319b00, {0xc02fb69600, 0x2b, 0x67}, {0xc073186380, 0x51c9860, 0xc073186380}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc048319b00, {0xc02fb69600, 0x2b, 0x77}, {0xc073186380, 0x2b, 0x67}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09284c398, {0xc02fb69600, 0x2b, 0x77}, {0xc073186380, 0x2b, 0x67}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09284acc0, {0xc073186380, 0x67, 0x77}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09284acc0, {0xc02fb69600, 0x2b, 0xc073186380}, {0xc073186380, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09284aa00, {0xc073186380, 0x2b, 0x67}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06c673b88, {{0xc023693770, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc00544ed00, {{0xc023693770, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05195d400, {0x63c85b8, 0xc0518b5fb0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0518b5fb0}, {0x64a6560, 0xc00544ed00}, {0xc023693770, 0x21036, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0518b5fb0}, {0x6466ed8, 0xc06c673b80}, {0xc023693770, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0873603a8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04d6f6300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020edcc00, {0x63c85b8, 0xc0518b5fb0}, 0xc04d6f6300, 0xc0413b81e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020edcc00, {0x63c85b8, 0xc0518b5fb0}, 0xc04d6f6300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0518b5ef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0518b5ef0}, {0xc0032fc8f0, 0x660089}, 0xc0032fc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0518b5ef0}, {0xc00102e4e0}, 0xc04d6f6200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0518b5e90}, 0xc04d6f6200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0bb6394d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808359 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc04aa59950}, 0xc05bf87368, 0x1, 0xc00e4710a8, 0x519bbc, 0xc05af6ede0, 0xc05905cba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04aa59950}, 0x120, 0xc00e471230, 0x5af6ed80, 0x0, 0x660069, 0xc05af6ede0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc04aa59950}, 0xc05af6ed80, {{{{0x0, 0x0}, {0xc029939680, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04aa59950}, 0x6352410, 0x53d3d378) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc04aa59950}, {0xc05af6ec60, {0x174301105c286a81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc04aa59950}, 0xc0b68cb4a0, 0xc00e4719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04aa59950}, 0x660089, {0xc05af6ec60, {0x174301105c286a81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc04aa59950}, 0xc04f057a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c286a81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c286a81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04aa598c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aa598c0}, {0xc00e4733f8, 0x660089}, 0xc00e473400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04aa598c0}, {0x3}, 0xc04f057700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04aa59830}, 0xc04f057700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}, 0xc0645df300, 0xc020326840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}, 0x3, 0xc011aba840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e473980}}}, {0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}, 0xc0645df300, 0xc011aba840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04aa59830}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aa59830}, {0x40, 0xc00e473b70}, 0xc00e473ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04aa59830}, {0x4e9d7a0, 0xc04f057700}, 0xc0645df300, 0xc020326840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04aa59830}, 0xc0a59391a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ac32a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ac32a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802510 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc06c7d7440}, 0xc065278500, 0xc04dab9cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc06c7d7440}, 0xc065278500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011023935546, 0x0, 0x0}, 0xc03d81cc80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011023935546, 0x0, 0x0}, 0xc03d81cc80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06c7d73b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c7d73b0}, {0xc0608bd3f8, 0x660089}, 0xc0608bd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06c7d73b0}, {0x656422202c343132}, 0xc065278300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06c7d7320}, 0xc065278300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}, 0xc058ed1940, 0xc02b55ff50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}, 0x3, 0xc06f2af100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0608bd980}}}, {0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}, 0xc058ed1940, 0xc06f2af100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06c7d7320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c7d7320}, {0x40, 0xc0608bdb70}, 0xc0608bdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06c7d7320}, {0x4e9d7a0, 0xc065278300}, 0xc058ed1940, 0xc02b55ff50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06c7d7320}, 0xc02c1e1ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04ace3440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04ace3440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812290 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc016c4d000, {0x63c85b8, 0xc01e08e8a0}, 0xc059187000, 0xc04646adc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc01e08e8a0}, 0xc059187000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e08e810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e08e810}, {0xc035c933f8, 0x660089}, 0xc035c93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e08e810}, {0xc0015221e0}, 0xc059186d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e08e780}, 0xc059186d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}, 0xc078fbc280, 0xc047c09758) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}, 0x3, 0xc039953000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc035c93980}}}, {0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}, 0xc078fbc280, 0xc039953000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01e08e780}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e08e780}, {0x40, 0xc035c93b70}, 0xc035c93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01e08e780}, {0x4e9d7a0, 0xc059186d00}, 0xc078fbc280, 0xc047c09758) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01e08e780}, 0xc04ca3b920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06d030fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06d030fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800598 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc018573000, {0x63c85b8, 0xc04760e3f0}, 0xc0af291200, 0xc04ceadd60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018573000, {0x63c85b8, 0xc04760e3f0}, 0xc0af291200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110151a7490, 0x0, 0x0}, 0xc08779bfa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110151a7490, 0x0, 0x0}, 0xc08779bfa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04760e360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04760e360}, {0xc076589918, 0x660089}, 0xc076589920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04760e360}, {0xc00102e4e0}, 0xc0af291100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04760e300}, 0xc0af291100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc0a9df2d80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04760e270}, {0xc00dfd36b0, 0x24}, 0x47d9c95b3, 0xc015271d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x4c1af3}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04760e270}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809953 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc061cbac00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812529 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812615 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc0304ca660}, 0xc031cfb380, 0x1, 0x7080183b7265b7, 0x37e11d600, 0xc016d8f080, 0xc0765b28a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05afa9540, {0x63c85b8, 0xc0304ca660}, 0xc0105fb320, 0xc0422851a8, 0x16d8f080, 0x0, 0x0, 0xc016d8f080, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc0304ca660}, 0xc016d8efc0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc02f72e900, 0x54}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0304ca660}, 0x6352260, 0x47c52620) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc0304ca660}, {0xc016d8ed80, {0x174301105981709e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc0304ca660}, 0xc01e1e9b80, 0xc0422859f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0304ca660}, 0x660089, {0xc016d8ed80, {0x174301105981709e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0304ca660}, 0xc0460dea00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105981709e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105981709e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0304ca570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304ca570}, {0xc0422873f8, 0x660089}, 0xc042287400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0304ca570}, {0xa1c0a1ff90b9848b}, 0xc0460de800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0304ca450}, 0xc0460de800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}, 0xc03ebdd780, 0xc031cfb350) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}, 0x3, 0xc015b9e080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042287980}}}, {0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}, 0xc03ebdd780, 0xc015b9e080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0304ca450}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304ca450}, {0x40, 0xc042287b70}, 0xc042287ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0304ca450}, {0x4e9d7a0, 0xc0460de800}, 0xc03ebdd780, 0xc031cfb350) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0304ca450}, 0xc039f0b560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810388 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bca4c00, {0x63c85b8, 0xc03bf1c180}, 0xc08ad6f300, 0xc08b787360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bca4c00, {0x63c85b8, 0xc03bf1c180}, 0xc08ad6f300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cef0a2e, 0x0, 0x0}, 0xc03905a700, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cef0a2e, 0x0, 0x0}, 0xc03905a700, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03bf1c0f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bf1c0f0}, {0xc0293393f8, 0x660089}, 0xc029339400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03bf1c0f0}, {0x7fe42934fa20}, 0xc08ad6f100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03bf1c060}, 0xc08ad6f100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}, 0xc07252ab80, 0xc03d272b70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}, 0x3, 0xc073feb0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc029339980}}}, {0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}, 0xc07252ab80, 0xc073feb0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03bf1c060}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bf1c060}, {0x40, 0xc029339b70}, 0xc029339ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03bf1c060}, {0x4e9d7a0, 0xc08ad6f100}, 0xc07252ab80, 0xc03d272b70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03bf1c060}, 0xc04d653980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc066b67200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc066b67200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809179 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b9a2f00, {0x63c85b8, 0xc06bfdc8d0}, 0xc0573a85e8, 0x1, 0xc005ee50a8, 0x519bbc, 0xc01ea2aba0, 0xc09acade60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06bfdc8d0}, 0x1e0, 0xc005ee5230, 0x1ea2ab40, 0x0, 0x660069, 0xc01ea2aba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b9a2f00, {0x63c85b8, 0xc06bfdc8d0}, 0xc01ea2ab40, {{{{0x0, 0x0}, {0xc0118c8240, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06bfdc8d0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0fbb30, {0x63c85b8, 0xc06bfdc8d0}, {0xc05529b320, {0x17430110644ab758, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0fbb30, {0x63c85b8, 0xc06bfdc8d0}, 0xc0517137c0, 0xc005ee59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06bfdc8d0}, 0x660089, {0xc05529b320, {0x17430110644ab758, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9bd400, {0x63c85b8, 0xc06bfdc8d0}, 0xc05bac1f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110644ab758, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110644ab758, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06bfdc840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06bfdc840}, {0xc005ee73f8, 0x660089}, 0xc005ee7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06bfdc840}, {0xa94ab0be3b0cf543}, 0xc05bac1d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06bfdc7b0}, 0xc05bac1d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}, 0xc06db6e300, 0xc074a62780) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}, 0x3, 0xc051842e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ee7980}}}, {0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}, 0xc06db6e300, 0xc051842e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06bfdc7b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06bfdc7b0}, {0x40, 0xc005ee7b70}, 0xc005ee7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06bfdc7b0}, {0x4e9d7a0, 0xc05bac1d00}, 0xc06db6e300, 0xc074a62780) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06bfdc7b0}, 0xc09a064c60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05529b200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05529b200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219798720 [runnable]: sync.runtime_SemacquireMutex(0xc092ad6690, 0x8f, 0xc092ad66a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc092ad6538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc092ad6560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0585d0000, {0xffffffffffffff15, 0xc092ad66c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0585d0000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0585d0000, {0xc04b3ff180, 0x32, 0x6e}, {0xc0299822d0, 0x32, 0x88}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc08ffdfe98, {0xc04b3ff180, 0x32, 0x6e}, {0xc0299822d0, 0x32, 0x88}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc08ffde7c0, {0xc0299822d0, 0x88, 0x6e}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc08ffde7c0, {0xc04b3ff180, 0x88, 0xc0299822d0}, {0xc0299822d0, 0xc04d8fb590, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc08ffde500, {0xc0299822d0, 0x32, 0x88}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc072e259a8, {{0xc04d8fb590, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0618db400, {{0xc036924420, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0523c9c00, {0x63c85b8, 0xc01b2aa5a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01b2aa5a0}, {0x64a6560, 0xc0618db400}, {0xc036924420, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01b2aa5a0}, {0x6466ed8, 0xc072e25700}, {0xc036924420, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06fb9d878}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc061baa100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02057a400, {0x63c85b8, 0xc01b2aa5a0}, 0xc061baa100, 0xc04cee8aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02057a400, {0x63c85b8, 0xc01b2aa5a0}, 0xc061baa100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b2aa510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b2aa510}, {0xc003eea8f0, 0x660089}, 0xc003eea8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b2aa510}, {0x24}, 0xc061baa000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b2aa480}, 0xc061baa000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03c4c6d20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811083 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc017f14640, {0x63c85b8, 0xc02d617050}, 0xc046a94960, 0x1, 0xc00b7cf0a8, 0x519bbc, 0xc0531afc80, 0xc0ce793880) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0731326c0, {0x63c85b8, 0xc02d617050}, 0xc0731326f0, 0xc00b7cf1a8, 0x16, 0x18, 0xffffffffffffffff, 0xc0531afc80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc017f14640, {0x63c85b8, 0xc02d617050}, 0xc0531afc20, {{{{0x0, 0x0}, {0xc073a70ea0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02d617050}, 0x6352230, 0x51b89dc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc017e6e550, {0x63c85b8, 0xc02d617050}, {0xc0531afb00, {0x17430110725c1c81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc017e6e550, {0x63c85b8, 0xc02d617050}, 0xc06c7bcbe0, 0xc00b7cf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02d617050}, 0x660089, {0xc0531afb00, {0x17430110725c1c81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017f21400, {0x63c85b8, 0xc02d617050}, 0xc084d72900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110725c1c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110725c1c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02d616e10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d616e10}, {0xc00b7d13f8, 0x660089}, 0xc00b7d1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02d616e10}, {0x7fe3953537d0}, 0xc084d72700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02d616b70}, 0xc084d72700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}, 0xc03e8a0120, 0xc04bc54c90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}, 0x3, 0xc0202e4bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b7d1980}}}, {0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}, 0xc03e8a0120, 0xc0202e4bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02d616b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d616b70}, {0x40, 0xc00b7d1b70}, 0xc00b7d1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02d616b70}, {0x4e9d7a0, 0xc084d72700}, 0xc03e8a0120, 0xc04bc54c90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02d616b70}, 0xc04f7e61e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0484c8240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0484c8240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802229 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc053d5fa40}, 0xc0a76f4100, 0xc02a1a3d60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc053d5fa40}, 0xc0a76f4100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102167c534, 0x0, 0x0}, 0xc043cdbf90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102167c534, 0x0, 0x0}, 0xc043cdbf90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053d5f9b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053d5f9b0}, {0xc03e6b33f8, 0x660089}, 0xc03e6b3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053d5f9b0}, {0x7fe38e297da8}, 0xc07e271f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053d5f920}, 0xc07e271f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}, 0xc06b394940, 0xc047794120) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}, 0x3, 0xc0474f4800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03e6b3980}}}, {0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}, 0xc06b394940, 0xc0474f4800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc053d5f920}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053d5f920}, {0x40, 0xc03e6b3b70}, 0xc03e6b3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc053d5f920}, {0x4e9d7a0, 0xc07e271f00}, 0xc06b394940, 0xc047794120) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc053d5f920}, 0xc076e8f920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06c6c0480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06c6c0480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811001 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc0571b7f20}, 0xc02632ea80, 0x1, 0x0, 0x1, 0xc00a3918c0, 0xc09b274f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0571b7f20}, 0x1e0, 0xc00ecbe728, 0xa391860, 0x0, 0x660069, 0xc00a3918c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc0571b7f20}, 0xc00a391860, {{{{0x0, 0x0}, {0xc06d6e4b40, 0x3e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc082446840, {0x63c85b8, 0xc0571b7f20}, 0xc0256aa610, 0x81e5c920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc0571b7f20}, {0xc00b8b1320, {0x1743011065e70dad, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc0571b7f20}, 0xc02c9ff9a0, 0xc081e5cee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0571b7f20}, 0x660089, {0xc00b8b1320, {0x1743011065e70dad, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0571b7f20}, 0xc076407300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065e70dad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065e70dad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0571b7e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0571b7e90}, {0xc0076ae8f0, 0x660089}, 0xc0076ae8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0571b7e90}, {0xc00102e4e0}, 0xc076407200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0571b7e30}, 0xc076407200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06dca1e90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811907 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc024a55ce0}, 0xc04479af48, 0x1, 0x70801818271cad, 0x37e11d600, 0xc06fafc180, 0xc0b9caf800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc024a55ce0}, 0x120, 0xc032d96178, 0x6fafc120, 0x0, 0x660069, 0xc06fafc180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc024a55ce0}, 0xc06fafc120, {{{{0x0, 0x0}, {0xc0a432cd80, 0x11b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc024a55ce0}, 0x6352410, 0x3c90c090) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc024a55ce0}, {0xc06fafc000, {0x174301107747cda6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc024a55ce0}, 0xc03b365040, 0xc07d278938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024a55ce0}, 0x660089, {0xc06fafc000, {0x174301107747cda6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc024a55ce0}, 0xc010bc4800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107747cda6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107747cda6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024a55c20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024a55c20}, {0xc07d27a340, 0x660089}, 0xc07d27a348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024a55c20}, {0xc00102e4e0}, 0xc010bc4700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024a55bc0}, 0xc010bc4700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc01c2f12f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810818 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06c800, {0x63c85b8, 0xc0880e3230}, 0xc062d47b00, 0xc03c8490e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc0880e3230}, 0xc062d47b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d8bb05b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d8bb05b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0880e3050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0880e3050}, {0xc01aa513f8, 0x660089}, 0xc01aa51400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0880e3050}, {0x0}, 0xc062d47900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086e8ff80}, 0xc062d47900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}, 0xc020f94760, 0xc076f9e960) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}, 0x3, 0xc039cbad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aa51980}}}, {0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}, 0xc020f94760, 0xc039cbad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086e8ff80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086e8ff80}, {0x40, 0xc01aa51b70}, 0xc01aa51ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086e8ff80}, {0x4e9d7a0, 0xc062d47900}, 0xc020f94760, 0xc076f9e960) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086e8ff80}, 0xc0d0782480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041f2c120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041f2c120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810124 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc024f2e7b0}, {0xc081c88700, 0x1, 0xc01f00cb30}, {0x1, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntent(0x0, {0x63c85b8, 0xc024f2e7b0}, {{{0xc02b8287b0, 0x22, 0x30}, {0x0, 0x0, 0x0}}, {{0xb0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:841 +0x9b github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc01d7f4f50, {0x63c85b8, 0xc024f2e7b0}, {0xc06acdfb00, {0x174301106ba5a231, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:549 +0x6db github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc024f2e7b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc01d7f4f50, {0x63c85b8, 0xc024f2e7b0}, {0xc06acdfb00, {0x174301106ba5a231, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc024f2e7b0}, 0xc064a9c3c0, 0xc081c89558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024f2e7b0}, 0xc064a9c3c0, {0xc06acdfb00, {0x174301106ba5a231, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc024f2e7b0}, 0xc051d53800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106ba5a231, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106ba5a231, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024f2e720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024f2e720}, {0xc00ecc13f8, 0x660089}, 0xc00ecc1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024f2e720}, {0x7fe351f74048}, 0xc051d53600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024f2e690}, 0xc051d53600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}, 0xc0198dbde0, 0xc0462b5f50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}, 0x3, 0xc074747f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ecc1980}}}, {0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}, 0xc0198dbde0, 0xc074747f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc024f2e690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024f2e690}, {0x40, 0xc00ecc1b70}, 0xc00ecc1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc024f2e690}, {0x4e9d7a0, 0xc051d53600}, 0xc0198dbde0, 0xc0462b5f50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc024f2e690}, 0xc09b2d6720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06acdf8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06acdf8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805672 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc060c8fe60}, 0xc09c8cc3c0, 0x1, 0xc00386f0a8, 0x519bbc, 0xc065c90b60, 0xc06e5d2660) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0520558c0, {0x63c85b8, 0xc060c8fe60}, 0xc052055908, 0xc00386f1a8, 0x40, 0x60, 0xffffffffffffffff, 0xc065c90b60, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc060c8fe60}, 0xc0499e0720, {{{{0xc0a54c4a20, 0x6}, {0xc08374ea20, 0x6}}, {{0xc0671230e0, 0x14}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc060c8fe60}, 0x63521a0, 0x1cdd9588) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc060c8fe60}, {0xc00f1030e0, {0x1743010fff544bb5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc060c8fe60}, 0xc06034b860, 0xc00386f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc060c8fe60}, 0x660089, {0xc00f1030e0, {0x1743010fff544bb5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc060c8fe60}, 0xc04ae25700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fff544bb5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fff544bb5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060c8fdd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060c8fdd0}, {0xc0038713f8, 0x660089}, 0xc003871400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060c8fdd0}, {0x7fe429e510d0}, 0xc04ae25400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc060c8fd10}, 0xc04ae25400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}, 0xc06deb8060, 0xc07d344258) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}, 0x3, 0xc092d4aec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003871980}}}, {0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}, 0xc06deb8060, 0xc092d4aec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc060c8fd10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060c8fd10}, {0x40, 0xc003871b70}, 0xc003871ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc060c8fd10}, {0x4e9d7a0, 0xc04ae25400}, 0xc06deb8060, 0xc07d344258) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc060c8fd10}, 0xc0499e05a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08bc15d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08bc15d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218864886 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0608d6990, 0xd) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0d8486000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0d8486660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0871ad900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0871ad900, {0x63c8510, 0xc0608d69c0}, 0xc00d7a1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06651c8a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0608d69c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06651c8a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219798406 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc016b9a3c0}, 0xc034639cb0, {0x646cc48, 0xc016b9a420}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc016b9a3c0}, 0xedb7a834e, 0xc07c4a90e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc016b9a3c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0b7932180}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc034639ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0b7932180}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 206962021 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99008, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc072556800, 0xc035925800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc072556800, {0xc035925800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc072556800, {0xc035925800, 0x5, 0xc0794febc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc055c37430, {0xc035925800, 0x80, 0x6}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0482f2a80, {0xc035925800, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04434fc50, {0xc035925800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0436b7af8, {0x631b900, 0xc04434fc50}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0436b7880, {0x7fe42ab59310, 0xc0482f2a80}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0436b7880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0436b7880, {0xc0a4abc000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02c6552d8, {0xc0a4abc000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06c7b3588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06c7b3588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06c7b3b60, {0x634aac0, 0xc06c7b3588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06c7b3500, 0xc082e4a500, 0xc0794ff42b, 0xc0794ff42d, {0x635bec0, 0xc03f157c20}, 0xc000d6ae00, 0xc00ecd7688, 0xc0794ff429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06c7b3500, {0x63c85b8, 0xc02d0dee70}, 0xc082e4a4f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a4aba009, 0x8}}, 0x0, 0xc02d0dec00, 0xc02d0dec30, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02d0dee70}, {0x643d810, 0xc0482f2a80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc039090840}, {0x643d810, 0xc0482f2a80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc039090840}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 206718697 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab97a90, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc068243d00, 0xc03f1ef000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc068243d00, {0xc03f1ef000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc068243d00, {0xc03f1ef000, 0x735, 0xc04c282a00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0634afc10, {0xc03f1ef000, 0x140, 0x13f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04e71d080, {0xc03f1ef000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02902d140, {0xc03f1ef000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03bbca5f8, {0x631b900, 0xc02902d140}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03bbca380, {0x7fe42ab59310, 0xc04e71d080}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03bbca380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03bbca380, {0xc00dfba000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc079f1a258, {0xc00dfba000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03fdb2088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03fdb2088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03fdb2660, {0x634aac0, 0xc03fdb2088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03fdb2000, 0xc00f1238f0, 0xc04cbd142b, 0xc04cbd142d, {0x635bec0, 0xc057bd1170}, 0xc000d6ae00, 0xc041b70c38, 0xc04cbd1429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03fdb2000, {0x63c85b8, 0xc020b17440}, 0xc00f1238e0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08e05c035, 0x8}}, 0x0, 0xc020b173e0, 0xc020b17410, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc020b17440}, {0x643d810, 0xc04e71d080}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06c8cf650}, {0x643d810, 0xc04e71d080}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06c8cf650}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808945 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc0176d6de0}, 0xc051eb0d00, 0xc0686110e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0176d6de0}, 0xc051eb0d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110618a8100, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110618a8100, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0176d6d20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0176d6d20}, {0xc00727d3f8, 0x660089}, 0xc00727d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0176d6d20}, {0x5}, 0xc051eb0b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0176d6c90}, 0xc051eb0b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}, 0xc0ce314ba0, 0xc0681a3578) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}, 0x3, 0xc043035f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00727d980}}}, {0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}, 0xc0ce314ba0, 0xc043035f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0176d6c90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0176d6c90}, {0x40, 0xc00727db70}, 0xc00727dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0176d6c90}, {0x4e9d7a0, 0xc051eb0b00}, 0xc0ce314ba0, 0xc0681a3578) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0176d6c90}, 0xc04de172c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081f6f9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081f6f9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805293 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc064a8bc20}, 0xc074bb8420, 0x1, 0xc0975cb0a8, 0x519bbc, 0xc05a2fc9c0, 0xc05827e420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc064a8bc20}, 0x1e0, 0xc0975cb230, 0x5a2fc960, 0xc068741380, 0x18, 0xc05a2fc9c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc064a8bc20}, 0xc05a2fc960, {{{{0x0, 0x0}, {0xc057fd3200, 0x43}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc064a8bc20}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc064a8bc20}, {0xc05ace38c0, {0x174301103edcf633, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc064a8bc20}, 0xc0296680a0, 0xc0975cb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc064a8bc20}, 0x660089, {0xc05ace38c0, {0x174301103edcf633, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc064a8bc20}, 0xc022096300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103edcf633, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103edcf633, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064a8bb90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064a8bb90}, {0xc0975cd3f8, 0x660089}, 0xc0975cd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064a8bb90}, {0x7fe4287b13c0}, 0xc022096100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064a8bb00}, 0xc022096100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}, 0xc01ae855e0, 0xc074bb8e88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}, 0x3, 0xc063739fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0975cd980}}}, {0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}, 0xc01ae855e0, 0xc063739fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc064a8bb00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064a8bb00}, {0x40, 0xc0975cdb70}, 0xc0975cdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}, 0x203018, 0x203018) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc064a8bb00}, {0x4e9d7a0, 0xc022096100}, 0xc01ae855e0, 0xc074bb8e88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc064a8bb00}, 0xc039164000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ace37a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ace37a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809812 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0345329f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219773310 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa04b8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc08558bf00, 0xc016b4a800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc08558bf00, {0xc016b4a800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc08558bf00, {0xc016b4a800, 0x735, 0xc0559e2640}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03eaa0620, {0xc016b4a800, 0x7eda79, 0xc016b4a805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0296793e0, {0xc016b4a800, 0xc078f96c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc021b9bec0, {0xc016b4a800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0781bde78, {0x631b900, 0xc021b9bec0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0781bdc00, {0x7fe42ab59310, 0xc0296793e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0781bdc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0781bdc00, {0xc0b0035000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc020fb6090, {0xc0b0035000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc08ec6b588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc08ec6b588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc08ec6bb60, {0x634aac0, 0xc08ec6b588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc08ec6b500, 0xc0415168b0, 0xc078f9742b, 0xc078f9742d, {0x635bec0, 0xc06485d290}, 0xc000d6ae00, 0xc01abee664, 0xc078f97429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc08ec6b500, {0x63c85b8, 0xc02217e6f0}, 0xc0415168a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc037946009, 0x8}}, 0x0, 0xc02217e690, 0xc02217e6c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02217e6f0}, {0x643d810, 0xc0296793e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07a08bf50}, {0x643d810, 0xc0296793e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07a08bf50}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803880 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021ad2c80, {0x63c85b8, 0xc06dcbb650}, 0xc031738f30, 0x1, 0xc07642f0a8, 0x519bbc, 0xc04cdc4960, 0xc064a87a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06dcbb650}, 0x120, 0xc07642f230, 0x4cdc4900, 0x0, 0x660069, 0xc04cdc4960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021ad2c80, {0x63c85b8, 0xc06dcbb650}, 0xc04cdc4900, {{{{0x0, 0x0}, {0xc0d43f0ea0, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06dcbb650}, 0x6352410, 0x45bf4b30) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a319a0, {0x63c85b8, 0xc06dcbb650}, {0xc04cdc47e0, {0x1743011016897f20, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a319a0, {0x63c85b8, 0xc06dcbb650}, 0xc05f31c460, 0xc07642f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06dcbb650}, 0x660089, {0xc04cdc47e0, {0x1743011016897f20, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c06000, {0x63c85b8, 0xc06dcbb650}, 0xc06fe82000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011016897f20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011016897f20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06dcbb5c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dcbb5c0}, {0xc0764313f8, 0x660089}, 0xc076431400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06dcbb5c0}, {0xc04b7db0e0}, 0xc0651b5e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06dcbb530}, 0xc0651b5e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}, 0xc01a509220, 0xc04b9a68d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}, 0x3, 0xc027c35f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076431980}}}, {0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}, 0xc01a509220, 0xc027c35f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06dcbb530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dcbb530}, {0x40, 0xc076431b70}, 0xc076431ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06dcbb530}, {0x4e9d7a0, 0xc0651b5e00}, 0xc01a509220, 0xc04b9a68d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06dcbb530}, 0xc02d961f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04cdc46c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04cdc46c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802141 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc03fae9fb0}, 0xc062530600, 0xc06891a000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc03fae9fb0}, 0xc062530600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110224549cb, 0x0, 0x0}, 0xc048a98a80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110224549cb, 0x0, 0x0}, 0xc048a98a80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03fae9f20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fae9f20}, {0xc05350d3f8, 0x660089}, 0xc05350d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03fae9f20}, {0xc0015221e0}, 0xc062530400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03fae9e90}, 0xc062530400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}, 0xc039a9e480, 0xc0a63e9e78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}, 0x3, 0xc03f581180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05350d980}}}, {0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}, 0xc039a9e480, 0xc03f581180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03fae9e90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fae9e90}, {0x40, 0xc05350db70}, 0xc05350dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03fae9e90}, {0x4e9d7a0, 0xc062530400}, 0xc039a9e480, 0xc0a63e9e78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03fae9e90}, 0xc050ea5b00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc01c0b06c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc01c0b06c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808119 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc05387fcc0, {0x63c85b8, 0xc06af68de0}, 0xc022738ed0, 0x1, 0xc0070f30a8, 0x519bbc, 0xc051e10540, 0xc05ee14f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc051fed7a0, {0x63c85b8, 0xc06af68de0}, 0xc051e10540, 0xc0070f31a8, 0x38, 0x40, 0x0, 0xc051e10540, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc05387fcc0, {0x63c85b8, 0xc06af68de0}, 0xc051e10480, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc073da7560, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06af68de0}, 0x6352260, 0x1b929750) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc051ee6280, {0x63c85b8, 0xc06af68de0}, {0xc051e10360, {0x174301101f4b11dc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc051ee6280, {0x63c85b8, 0xc06af68de0}, 0xc0472092c0, 0xc0070f39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06af68de0}, 0x660089, {0xc051e10360, {0x174301101f4b11dc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05004cc00, {0x63c85b8, 0xc06af68de0}, 0xc03d86ae00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101f4b11dc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101f4b11dc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06af68d50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06af68d50}, {0xc0070f53f8, 0x660089}, 0xc0070f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06af68d50}, {0x0}, 0xc03d86ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06af68cc0}, 0xc03d86ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}, 0xc058ddbe80, 0xc029bcd398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}, 0x3, 0xc01bedd940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0070f5980}}}, {0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}, 0xc058ddbe80, 0xc01bedd940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06af68cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06af68cc0}, {0x40, 0xc0070f5b70}, 0xc0070f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06af68cc0}, {0x4e9d7a0, 0xc03d86ac00}, 0xc058ddbe80, 0xc029bcd398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06af68cc0}, 0xc041bef380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc051e10240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc051e10240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806833 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc090d0ff80}, 0xc088194cb0, {0x646cc48, 0xc08ff88000}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc090d0ff80}, 0xedb7a834f, 0xc050aa59e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc090d0ff80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc024cc6840}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc088194ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc024cc6840}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808821 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff69e00, {0x63c85b8, 0xc03137b200}, 0xc06d54ac78, 0x1, 0xc0058b90a8, 0x519bbc, 0xc073fdf260, 0xc0bde5e760) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03137b200}, 0x120, 0xc0058b9230, 0x73fdf200, 0x0, 0x660069, 0xc073fdf260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff69e00, {0x63c85b8, 0xc03137b200}, 0xc073fdf200, {{{{0x0, 0x0}, {0xc05907cd80, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03137b200}, 0x63526b0, 0x3f8ea6b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030006be0, {0x63c85b8, 0xc03137b200}, {0xc073fdf0e0, {0x174301105fed2503, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030006be0, {0x63c85b8, 0xc03137b200}, 0xc067026500, 0xc0058b99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03137b200}, 0x660089, {0xc073fdf0e0, {0x174301105fed2503, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030040800, {0x63c85b8, 0xc03137b200}, 0xc060229300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105fed2503, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105fed2503, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03137b170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03137b170}, {0xc0058bb3f8, 0x660089}, 0xc0058bb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03137b170}, {0x7fe351f778d8}, 0xc060229100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03137b0b0}, 0xc060229100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}, 0xc061a4f440, 0xc0158260c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}, 0x3, 0xc055a79080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0058bb980}}}, {0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}, 0xc061a4f440, 0xc055a79080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03137b0b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03137b0b0}, {0x40, 0xc0058bbb70}, 0xc0058bbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03137b0b0}, {0x4e9d7a0, 0xc060229100}, 0xc061a4f440, 0xc0158260c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03137b0b0}, 0xc0129ee000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc073fdeea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc073fdeea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799950 [runnable]: sync.runtime_Semrelease(0x3, 0x0, 0x0) GOROOT/src/runtime/sema.go:66 +0x19 sync.(*Mutex).unlockSlow(0x0, 0x0) GOROOT/src/sync/mutex.go:224 +0x65 sync.(*Mutex).Unlock(...) GOROOT/src/sync/mutex.go:190 sync.(*RWMutex).Unlock(0xc00074a760) GOROOT/src/sync/rwmutex.go:149 +0x9b github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fdb7530e000, 0x7d6e, 0x7fdb0ad5a8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:206 +0x2fc github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc03fbdcb00, 0x7fdb7530e001, 0x7fdb7530e020, 0x2bb5, 0x2bba) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0586cedc0, {0x5458e6, 0x2bb5}, 0x0, 0xc083793e78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc083793c00, {0xffffffffffffff15, 0xc091cf45e0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc083793c00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc083793c00, {0xc0bf863b90, 0x2b, 0x6f}, 0xc017ccce00, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc083793c00, {0xc017ccce00, 0x2b, 0x71}, {0xc0bf863b90, 0x2b, 0x6f}, 0x1, 0x25) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0853423e8, {0xc017ccce00, 0x1, 0x6f}, {0xc0bf863b90, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0853423e8, {0xc017ccce00, 0x2b, 0x71}, {0xc0bf863b90, 0x2b, 0x6f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0853411c0, {0xc0bf863b90, 0x6f, 0x71}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0853411c0, {0xc017ccce00, 0x2b, 0xc0bf863b90}, {0xc0bf863b90, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc085340f00, {0xc0bf863b90, 0x2b, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06060eb08, {{0xc08dc6b170, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc057871e00, {{0xc08dc6b170, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc067a37c00, {0x63c85b8, 0xc024122a80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc024122a80}, {0x64a6560, 0xc057871e00}, {0xc08dc6b170, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc024122a80}, {0x6466ed8, 0xc06060eb00}, {0xc08dc6b170, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06511aea0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc035f8fc00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209f1000, {0x63c85b8, 0xc024122a80}, 0xc035f8fc00, 0xc067f6cd20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209f1000, {0x63c85b8, 0xc024122a80}, 0xc035f8fc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0241229c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0241229c0}, {0xc0030568f0, 0x660089}, 0xc0030568f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0241229c0}, {0xc00102e4e0}, 0xc035f8fb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0241228d0}, 0xc035f8fb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc024122630}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812599 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804631 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc07c6a8ed0}, 0xc0a6c07d00, 0xc07839da40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07c6a8ed0}, 0xc0a6c07d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c6a8e40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c6a8e40}, {0xc0624453f8, 0x660089}, 0xc062445400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c6a8e40}, {0xc0015221e0}, 0xc0a6c07a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c6a8cc0}, 0xc0a6c07a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}, 0xc0406836c0, 0xc0175a0990) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}, 0x3, 0xc06c1e5000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062445980}}}, {0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}, 0xc0406836c0, 0xc06c1e5000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c6a8cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c6a8cc0}, {0x40, 0xc062445b70}, 0xc062445ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c6a8cc0}, {0x4e9d7a0, 0xc0a6c07a00}, 0xc0406836c0, 0xc0175a0990) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c6a8cc0}, 0xc043503ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc064009d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc064009d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810743 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02a71b800, {0x63c85b8, 0xc014b39980}, 0xc003e77400, 0xc02d348640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a71b800, {0x63c85b8, 0xc014b39980}, 0xc003e77400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f6f6e3a, 0x0, 0x0}, 0xc05f685ca0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f6f6e3a, 0x0, 0x0}, 0xc05f685ca0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc014b39470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc014b39470}, {0xc01fb273f8, 0x660089}, 0xc01fb27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc014b39470}, {0xc01e8eb000}, 0xc003e76c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc014b38f90}, 0xc003e76c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}, 0xc060f38080, 0xc048db1278) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}, 0x3, 0xc0436a1540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fb27980}}}, {0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}, 0xc060f38080, 0xc0436a1540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc014b38f90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc014b38f90}, {0x40, 0xc01fb27b70}, 0xc01fb27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc014b38f90}, {0x4e9d7a0, 0xc003e76c00}, 0xc060f38080, 0xc048db1278) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc014b38f90}, 0xc046001800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05ca83440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05ca83440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812556 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x80, 0xc01fce02c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc054078380, 0xc03baaf168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01fce02c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc054078380, {0xc042a8c240, 0x28, 0x90}, {0xc036c9ee70, 0x28, 0x6e}, 0x0, 0x6f) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0f1a26de0, {0xc042a8c240, 0x1, 0x6e}, {0xc036c9ee70, 0xc0f1a26000, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0f1a26de0, {0xc042a8c240, 0x28, 0x90}, {0xc036c9ee70, 0x28, 0x6e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0f1a262c0, {0xc036c9ee70, 0x0, 0xc03baaf4e0}, 0x121ea30, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0f1a262c0, {0xc042a8c240, 0x28, 0x6e}, {0xc036c9ee70, 0x27, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0f1a26000, {0xc036c9ee70, 0x28, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05820c588, {{0xc03a992cc0, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05779b500, {{0xc03a992cc0, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ae7f000, {0x63c85b8, 0xc056614e70}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056614e70}, {0x64a6560, 0xc05779b500}, {0xc03a992cc0, 0xc00bc1fa40, 0x30}, {0x174301109383f2b5, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056614e70}, {0x6466ed8, 0xc05820c580}, {0xc03a992cc0, 0x27, 0x30}, {0x174301109383f2b5, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07c51aa68}, {{0x174301109383f2b5, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109383f2b5, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046448100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b15157b5, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02f10ec00, {0x63c85b8, 0xc056614e70}, 0xc046448100, 0xc029669f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f10ec00, {0x63c85b8, 0xc056614e70}, 0xc046448100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109383f2b5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109383f2b5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056614de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614de0}, {0xc03bab53f8, 0x660089}, 0xc03bab5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056614de0}, {0x7fe13dc9e290}, 0xc083617e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056614d50}, 0xc083617e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}, 0xc0ccf7fda0, 0xc038161e48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}, 0x3, 0xc071e55080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03bab5980}}}, {0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}, 0xc0ccf7fda0, 0xc071e55080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056614d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614d50}, {0x40, 0xc03bab5b70}, 0xc03bab5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056614d50}, {0x4e9d7a0, 0xc083617e00}, 0xc0ccf7fda0, 0xc038161e48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056614d50}, 0xc0719830e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808168 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0838abc10, 0xd) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc006f4ca80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc006f4d0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0688b8c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0688b8c00, {0x63c8510, 0xc0838abc40}, 0xc08271ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03c3a08f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0838abc40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03c3a08f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219803448 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc092774cc0}, 0xc037e67248, 0x1, 0xc0755b70a8, 0x519bbc, 0xc0455c76e0, 0xc046fae420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc092774cc0}, 0x1e0, 0xc0755b7230, 0x455c7680, 0x0, 0x660069, 0xc0455c76e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc092774cc0}, 0xc0455c7680, {{{{0x0, 0x0}, {0xc079beab40, 0x42}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc092774cc0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc092774cc0}, {0xc03896f9e0, {0x174301102cc46955, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc092774cc0}, 0xc0750b5720, 0xc0755b79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc092774cc0}, 0x660089, {0xc03896f9e0, {0x174301102cc46955, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc092774cc0}, 0xc019298300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc46955, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc46955, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc092774c30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092774c30}, {0xc0755b93f8, 0x660089}, 0xc0755b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc092774c30}, {0x0}, 0xc019298100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc092774ba0}, 0xc019298100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}, 0xc0233dc800, 0xc034163f50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}, 0x3, 0xc01ee2fc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0755b9980}}}, {0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}, 0xc0233dc800, 0xc01ee2fc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc092774ba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092774ba0}, {0x40, 0xc0755b9b70}, 0xc0755b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc092774ba0}, {0x4e9d7a0, 0xc019298100}, 0xc0233dc800, 0xc034163f50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc092774ba0}, 0xc0260ce840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03896f8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03896f8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802776 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc050ed0c90}, 0xc08d16a348, 0x1, 0xc058efff20, 0x519bbc, 0xc072aab3e0, 0xc06f706d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc050ed0c90}, 0xc0, 0xc058f000a8, 0x72aab380, 0x0, 0x660069, 0xc072aab3e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc050ed0c90}, 0xc072aab380, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc050ed0c90}, 0x63524a0, 0x5591d270) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc050ed0c90}, {0x0, {0x174301105693a9c2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc050ed0c90}, 0xc04293ec80, 0xc058f00868) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc050ed0c90}, 0x660089, {0x0, {0x174301105693a9c2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc050ed0c90}, 0xc08ad6b200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050ed0c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050ed0c00}, {0xc058f02270, 0x660089}, 0xc058f02278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050ed0c00}, {0xc049cca930}, 0xc08ad6b100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050ed0ba0}, 0xc08ad6b100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x174301105693a9c2, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc03e83a000}, 0x0, 0xc011a38680) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc03e83a000}, 0x66040f) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc019919860, {0x63c85b8, 0xc03e83a000}, {{0x5c, 0x4, 0x96, 0xea, 0xaf, 0xab, 0x4b, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:922 +0x1b0 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc019919860, {0x63c85b8, 0xc03e83a000}, 0xc0157ceb40, 0xc072dd6d80, 0xc03d04ca00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:610 +0xd4a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc03e83a000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc049ccaa80}, {{0xc0651a1300, 0x24, 0x24}}, 0xc058f05380) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc019919860, {0x63c85b8, 0xc049ccaa80}, 0xc0157ceb40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0301af310, {0x63c85b8, 0xc049ccaa80}, {0x0, {0x1743011026889121, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc049ccaa80}, 0xc06c7e06e0, 0xc0308909d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc049ccaa80}, 0xc06c7e06e0, {0x0, {0x1743011026889121, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc049ccaa80}, 0xc048572200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011026889121, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011026889121, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011026889121, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc049cca9c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049cca9c0}, {0xc058e693f8, 0x660089}, 0xc058e69400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc049cca9c0}, {0xc0015221e0}, 0xc048572000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc049cca930}, 0xc048572000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}, 0xc040fb91c0, 0xc0798337a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}, 0x3, 0xc0b3a435c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc058e69980}}}, {0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}, 0xc040fb91c0, 0xc0b3a435c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc049cca930}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049cca930}, {0x40, 0xc058e69b70}, 0xc058e69ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc049cca930}, {0x4e9d7a0, 0xc048572000}, 0xc040fb91c0, 0xc0798337a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc049cca930}, 0xc02b325aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03c7eca20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03c7eca20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812851 [runnable]: sync.runtime_SemacquireMutex(0xc031fc9198, 0x8f, 0xc031fc91a8) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x1194e0b, 0xc031fc8ff8, 0x11948b8) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x41, 0x3d, 0x42, 0xc031fc9068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0547aa480, {0xffffffffffffff15, 0xc031fc91c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0547aa480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0547aa480, {0xc08e45e700, 0x30, 0x74}, {0xc02e4c1140, 0x30, 0x3d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093426d98, {0xc08e45e700, 0x30, 0x74}, {0xc02e4c1140, 0x30, 0x3d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0934256c0, {0xc02e4c1140, 0x3d, 0x74}, 0x30, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0934256c0, {0xc08e45e700, 0x3d, 0xc02e4c1140}, {0xc02e4c1140, 0xc0736500e0, 0x2f}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093425400, {0xc02e4c1140, 0x30, 0x3d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0564e7328, {{0xc0736500e0, 0x2f, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0061fb500, {{0xc08bd1fd10, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08c161800, {0x63c85b8, 0xc059b06510}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc059b06510}, {0x64a6560, 0xc0061fb500}, {0xc08bd1fd10, 0x4a, 0x30}, {0x174301109dd2151f, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc059b06510}, {0x6466ed8, 0xc0564e7080}, {0xc08bd1fd10, 0x28, 0x30}, {0x174301109dd2151f, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01b5ef020}, {{0x174301109dd2151f, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109dd2151f, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03e5b2000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bb9f7a1f, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc031009000, {0x63c85b8, 0xc059b06510}, 0xc03e5b2000, 0xc06f9665a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031009000, {0x63c85b8, 0xc059b06510}, 0xc03e5b2000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109dd2151f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109dd2151f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059b06450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059b06450}, {0xc031fcf3f8, 0x660089}, 0xc031fcf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059b06450}, {0x3fefffdc5f52ef36}, 0xc095dafd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc059b063c0}, 0xc095dafd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}, 0xc01b611ae0, 0xc0b2691d88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}, 0x3, 0xc0492823c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc031fcf980}}}, {0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}, 0xc01b611ae0, 0xc0492823c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc059b063c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059b063c0}, {0x40, 0xc031fcfb70}, 0xc031fcfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc059b063c0}, {0x4e9d7a0, 0xc095dafd00}, 0xc01b611ae0, 0xc0b2691d88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc059b063c0}, 0xc078dbc300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0b6a410e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0b6a410e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809619 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02499e8c0, {0x63c85b8, 0xc02109dd40}, 0xc08c139e18, 0x1, 0xc0279dd0a8, 0x519bbc, 0xc08a001b60, 0xc077e03ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02109dd40}, 0x120, 0xc0279dd230, 0x8a001b00, 0x0, 0x660069, 0xc08a001b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02499e8c0, {0x63c85b8, 0xc02109dd40}, 0xc08a001b00, {{{{0x0, 0x0}, {0xc04b78e120, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02109dd40}, 0x6352410, 0x4b1bcb70) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc023b3b7c0, {0x63c85b8, 0xc02109dd40}, {0xc08a0019e0, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc023b3b7c0, {0x63c85b8, 0xc02109dd40}, 0xc065099860, 0xc0279dd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02109dd40}, 0x660089, {0xc08a0019e0, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0249ba400, {0x63c85b8, 0xc02109dd40}, 0xc0600a9a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02109dc50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02109dc50}, {0xc0279df3f8, 0x660089}, 0xc0279df400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02109dc50}, {0x3ecf270a522403e2}, 0xc0600a9800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02109d8f0}, 0xc0600a9800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}, 0xc0405a3d40, 0xc03f1807f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}, 0x3, 0xc0be9c3680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0279df980}}}, {0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}, 0xc0405a3d40, 0xc0be9c3680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02109d8f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02109d8f0}, {0x40, 0xc0279dfb70}, 0xc0279dfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02109d8f0}, {0x4e9d7a0, 0xc0600a9800}, 0xc0405a3d40, 0xc03f1807f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02109d8f0}, 0xc03674bf80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc088226900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc088226900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806155 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc070c9d710}, 0xc08a455f68, 0x1, 0xc00437fff0, 0x519bbc, 0xc01bcdb6e0, 0xc05a1a76e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc070c9d710}, 0x1e0, 0xc004380178, 0x1bcdb680, 0x0, 0x660069, 0xc01bcdb6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc070c9d710}, 0xc01bcdb680, {{{{0x0, 0x0}, {0xc07d47de60, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc070c9d710}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc070c9d710}, {0xc044ddcc60, {0x174301104c82f06f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc070c9d710}, 0xc0734c0140, 0xc06f204938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc070c9d710}, 0x660089, {0xc044ddcc60, {0x174301104c82f06f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc070c9d710}, 0xc065023100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c82f06f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c82f06f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070c9d680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070c9d680}, {0xc06f206340, 0x660089}, 0xc06f206348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070c9d680}, {0xc00102e4e0}, 0xc065023000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070c9d620}, 0xc065023000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc04d6faba0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804442 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ef69e00, {0x63c85b8, 0xc03fe9fa10}, 0xc029f185b8, 0x1, 0xc07b23f0a8, 0x519bbc, 0xc04e360f60, 0xc04d6649c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03fe9fa10}, 0x1e0, 0xc07b23f230, 0x4e360f00, 0x0, 0x660069, 0xc04e360f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ef69e00, {0x63c85b8, 0xc03fe9fa10}, 0xc04e360f00, {{{{0x0, 0x0}, {0xc04a5645a0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03fe9fa10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01ef6ce60, {0x63c85b8, 0xc03fe9fa10}, {0xc05994cc60, {0x1743011032cd4a27, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01ef6ce60, {0x63c85b8, 0xc03fe9fa10}, 0xc05a5b01e0, 0xc07b23f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03fe9fa10}, 0x660089, {0xc05994cc60, {0x1743011032cd4a27, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f031000, {0x63c85b8, 0xc03fe9fa10}, 0xc054e6bd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032cd4a27, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032cd4a27, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03fe9f860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fe9f860}, {0xc07b2413f8, 0x660089}, 0xc07b241400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03fe9f860}, {0xc0015221e0}, 0xc054e6b900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03fe9f6b0}, 0xc054e6b900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}, 0xc055902c60, 0xc04dbe5d70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}, 0x3, 0xc0c3354ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b241980}}}, {0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}, 0xc055902c60, 0xc0c3354ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03fe9f6b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fe9f6b0}, {0x40, 0xc07b241b70}, 0xc07b241ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03fe9f6b0}, {0x4e9d7a0, 0xc054e6b900}, 0xc055902c60, 0xc04dbe5d70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03fe9f6b0}, 0xc03e903a40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05994c900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05994c900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219150686 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219805089 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc064cd7350}, 0xc042e8dbd8, 0x1, 0x0, 0x1, 0xc07279c060, 0xc01b5e63c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc064cd7350}, 0x120, 0xc078b12728, 0x7279c000, 0xc068a944e0, 0x18, 0xc07279c060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc064cd7350}, 0xc07279c000, {{{{0x0, 0x0}, {0xc055a9b200, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc064cd7350}, 0x6352410, 0x1f8e1588) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc064cd7350}, {0xc0722aea20, {0x174301101b746c81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc064cd7350}, 0xc02c6b41e0, 0xc075e6eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc064cd7350}, 0x660089, {0xc0722aea20, {0x174301101b746c81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc064cd7350}, 0xc055bf6a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b746c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b746c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064cd72c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064cd72c0}, {0xc0032fa8f0, 0x660089}, 0xc0032fa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064cd72c0}, {0xc00102e4e0}, 0xc055bf6900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064cd7260}, 0xc055bf6900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06c2f8d80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805079 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc077da5110}, 0xc0310518c0, 0x1, 0x0, 0x1, 0xc0720084e0, 0xc06e9fc3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077da5110}, 0x120, 0xc08908a728, 0x72008480, 0xc068a94680, 0x18, 0xc0720084e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc077da5110}, 0xc072008480, {{{{0x0, 0x0}, {0xc0557bc900, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077da5110}, 0x6352410, 0x1f8e1530) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc077da5110}, {0xc0722aea20, {0x174301101b746c81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc077da5110}, 0xc02044f360, 0xc07b0d4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077da5110}, 0x660089, {0xc0722aea20, {0x174301101b746c81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc077da5110}, 0xc0768c0100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b746c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b746c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077da5050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077da5050}, {0xc0036f68f0, 0x660089}, 0xc0036f68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077da5050}, {0x24}, 0xc0768c0000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077da4ff0}, 0xc0768c0000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06c2f89c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812598 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801773 [runnable]: sync.runtime_SemacquireMutex(0xc086a4c690, 0x8f, 0x7fdf852b8b40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc086a4c538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc086a4c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0849b9680, {0xffffffffffffff15, 0xc086a4c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0849b9680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0849b9680, {0xc02c709ec0, 0x32, 0x5f}, {0xc02354cb00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092eab998, {0xc02c709ec0, 0x32, 0x5f}, {0xc02354cb00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092eaa2c0, {0xc02354cb00, 0x72, 0x5f}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092eaa2c0, {0xc02c709ec0, 0x72, 0xc02354cb00}, {0xc02354cb00, 0xc032fb1e00, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092eaa000, {0xc02354cb00, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06c4e99a8, {{0xc032fb1e00, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0224a0e00, {{0xc079ce07b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc049a53000, {0x63c85b8, 0xc052e14ba0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc052e14ba0}, {0x64a6560, 0xc0224a0e00}, {0xc079ce07b0, 0x21036, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc052e14ba0}, {0x6466ed8, 0xc06c4e9700}, {0xc079ce07b0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07c51a720}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc077d4e300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020e15000, {0x63c85b8, 0xc052e14ba0}, 0xc077d4e300, 0xc0297bc960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020e15000, {0x63c85b8, 0xc052e14ba0}, 0xc077d4e300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc052e14b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052e14b10}, {0xc0032fc8f0, 0x660089}, 0xc0032fc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc052e14b10}, {0xc00102e4e0}, 0xc077d4e200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc052e14ab0}, 0xc077d4e200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618caf30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809795 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc012025000, {0x63c85b8, 0xc074868c00}, 0xc064499f00, 0xc0650995e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc012025000, {0x63c85b8, 0xc074868c00}, 0xc064499f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066f2f3a2, 0x0, 0x0}, 0xc04a850540, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066f2f3a2, 0x0, 0x0}, 0xc04a850540, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074868b70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074868b70}, {0xc01100f3f8, 0x660089}, 0xc01100f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074868b70}, {0xc01f093c00}, 0xc064499c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074868ae0}, 0xc064499c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}, 0xc0631ce540, 0xc066ab4a08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}, 0x3, 0xc017baf4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01100f980}}}, {0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}, 0xc0631ce540, 0xc017baf4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc074868ae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074868ae0}, {0x40, 0xc01100fb70}, 0xc01100fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc074868ae0}, {0x4e9d7a0, 0xc064499c00}, 0xc0631ce540, 0xc066ab4a08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc074868ae0}, 0xc0243ae900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07abd3200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07abd3200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805493 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc06ec16c60}, 0xc015d6bae8, 0x1, 0xc0834fb0a8, 0x519bbc, 0xc05b734ba0, 0xc0590bcba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06ec16c60}, 0x1e0, 0xc0834fb230, 0x5b734b40, 0xc067b54ea0, 0x18, 0xc05b734ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc06ec16c60}, 0xc05b734b40, {{{{0x0, 0x0}, {0xc057fd3200, 0x43}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06ec16c60}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc06ec16c60}, {0xc05b77efc0, {0x174301103fe19b73, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc06ec16c60}, 0xc04bcedb80, 0xc0834fb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06ec16c60}, 0x660089, {0xc05b77efc0, {0x174301103fe19b73, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc06ec16c60}, 0xc065241d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103fe19b73, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103fe19b73, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ec16bd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ec16bd0}, {0xc0834fd3f8, 0x660089}, 0xc0834fd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ec16bd0}, {0x0}, 0xc065241b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06ec16b40}, 0xc065241b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}, 0xc0724bc080, 0xc0641203a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}, 0x3, 0xc062e837c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0834fd980}}}, {0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}, 0xc0724bc080, 0xc062e837c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06ec16b40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ec16b40}, {0x40, 0xc0834fdb70}, 0xc0834fdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}, 0x203018, 0x203018) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06ec16b40}, {0x4e9d7a0, 0xc065241b00}, 0xc0724bc080, 0xc0641203a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06ec16b40}, 0xc0cd4e04e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05b77e7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05b77e7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219528831 [IO wait]: internal/poll.runtime_pollWait(0x7fdac11849e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc079716c00, 0xc027d9e000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc079716c00, {0xc027d9e000, 0x2b7d, 0x2b7d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc079716c00, {0xc027d9e000, 0x2b7d, 0xc03e634640}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0394f4d88, {0xc027d9e000, 0x7eda79, 0xc027d9e005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0496400c0, {0xc027d9e000, 0xc08f94ac60, 0x2b7d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2690ff0, {0xc027d9e000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0d05e2278, {0x631b900, 0xc0b2690ff0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0d05e2000, {0x7fe42ab59310, 0xc0496400c0}, 0x2b7d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0d05e2000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0d05e2000, {0xc01b446000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc07554b740, {0xc01b446000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06c282b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06c282b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06c2830e0, {0x634aac0, 0xc06c282b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06c282a80, 0xc01f1d68e0, 0xc09195b42b, 0xc09195b42d, {0x635bec0, 0xc017bff170}, 0xc000d6ae00, 0xc0305df038, 0xc09195b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06c282a80, {0x63c85b8, 0xc056fda330}, 0xc01f1d68d0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc085265037, 0x8}}, 0x0, 0xc056fda2d0, 0xc056fda300, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc056fda330}, {0x643d810, 0xc0496400c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc074bf6060}, {0x643d810, 0xc0496400c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc074bf6060}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805042 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03c908c80, {0x63c85b8, 0xc04a149950}, 0xc08afc6198, 0x1, 0xc07b5810a8, 0x519bbc, 0xc0703e6a80, 0xc05f2a3da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04a149950}, 0x120, 0xc07b581230, 0x703e6a20, 0xc067b551e0, 0x18, 0xc0703e6a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03c908c80, {0x63c85b8, 0xc04a149950}, 0xc0703e6a20, {{{{0x0, 0x0}, {0xc088ff2fc0, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04a149950}, 0x6352410, 0x52e78090) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00852fe50, {0x63c85b8, 0xc04a149950}, {0xc0703e6900, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00852fe50, {0x63c85b8, 0xc04a149950}, 0xc06c7bcaa0, 0xc07b5819f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04a149950}, 0x660089, {0xc0703e6900, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b078c00, {0x63c85b8, 0xc04a149950}, 0xc06f10af00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04a149830}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a149830}, {0xc07b5833f8, 0x660089}, 0xc07b583400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04a149830}, {0x4d939c}, 0xc06f10ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04a149740}, 0xc06f10ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}, 0xc028199ca0, 0xc08afc6690) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}, 0x3, 0xc083cd7c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b583980}}}, {0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}, 0xc028199ca0, 0xc083cd7c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04a149740}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a149740}, {0x40, 0xc07b583b70}, 0xc07b583ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}, 0x203020, 0x203020) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04a149740}, {0x4e9d7a0, 0xc06f10ad00}, 0xc028199ca0, 0xc08afc6690) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04a149740}, 0xc064bb0720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811866 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01de7ea00, {0x63c85b8, 0xc0654c1740}, 0xc034d3d488, 0x1, 0xc01f21d0a8, 0x519bbc, 0xc0469c44e0, 0xc0838a2de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0654c1740}, 0x120, 0xc01f21d230, 0x469c4480, 0x0, 0x660069, 0xc0469c44e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01de7ea00, {0x63c85b8, 0xc0654c1740}, 0xc0469c4480, {{{{0x0, 0x0}, {0xc0c8e82a20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0654c1740}, 0x6352410, 0x55ac41a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dec4000, {0x63c85b8, 0xc0654c1740}, {0xc0469c4360, {0x1743011065ea6c77, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dec4000, {0x63c85b8, 0xc0654c1740}, 0xc05292fb80, 0xc01f21d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0654c1740}, 0x660089, {0xc0469c4360, {0x1743011065ea6c77, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ded1400, {0x63c85b8, 0xc0654c1740}, 0xc06e40ed00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ea6c77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ea6c77, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0654c15c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0654c15c0}, {0xc01f21f3f8, 0x660089}, 0xc01f21f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0654c15c0}, {0x7fe3b59836f0}, 0xc06e40eb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0654c1530}, 0xc06e40eb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}, 0xc0428fd6c0, 0xc03f861f38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}, 0x3, 0xc0450f8ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f21f980}}}, {0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}, 0xc0428fd6c0, 0xc0450f8ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0654c1530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0654c1530}, {0x40, 0xc01f21fb70}, 0xc01f21fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0654c1530}, {0x4e9d7a0, 0xc06e40eb00}, 0xc0428fd6c0, 0xc03f861f38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0654c1530}, 0xc0bac657a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0469c4120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0469c4120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802565 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0934d2548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0934d25b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0f1bc45f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0be723c00, {0xc03ef52620, 0x2b, 0x68}, {0xc05fbb4c00, 0x2b, 0x57}, 0x1, 0x45) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0f1bc4488, {0xc03ef52620, 0x1, 0x57}, {0xc05fbb4c00, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0f1bc4488, {0xc03ef52620, 0x2b, 0x68}, {0xc05fbb4c00, 0x2b, 0x57}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0f1bc3bc0, {0xc05fbb4c00, 0x57, 0x68}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0f1bc3bc0, {0xc03ef52620, 0x2b, 0xc05fbb4c00}, {0xc05fbb4c00, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0f1bc3900, {0xc05fbb4c00, 0x2b, 0x57}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06559a588, {{0xc0859563c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06ef31900, {{0xc0859563c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0532bf000, {0x63c85b8, 0xc07660b7a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc07660b7a0}, {0x64a6560, 0xc06ef31900}, {0xc0859563c0, 0x20eda, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc07660b7a0}, {0x6466ed8, 0xc06559a580}, {0xc0859563c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0a63e9ad0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc062530a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f1cc00, {0x63c85b8, 0xc07660b7a0}, 0xc062530a00, 0xc04c4497c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f1cc00, {0x63c85b8, 0xc07660b7a0}, 0xc062530a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07660b6e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07660b6e0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07660b6e0}, {0xc00102e4e0}, 0xc062530900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07660b680}, 0xc062530900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0632207b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806586 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc08dd7c810}, 0xc01636bc68, 0x1, 0x708017ed9ffb0b, 0x37e11d600, 0xc0489c89c0, 0xc0489c87e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc08dd7c810}, 0x1e0, 0xc002d33230, 0x489c8960, 0x0, 0x660069, 0xc0489c89c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc08dd7c810}, 0xc0489c8960, {{{{0x0, 0x0}, {0xc093ef9680, 0x111}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc031e63aa0, {0x63c85b8, 0xc08dd7c810}, 0xc00eefb3d0, 0x2d33428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc08dd7c810}, {0xc014a46000, {0x174301104ef00b46, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc08dd7c810}, 0xc07a02abe0, 0xc002d339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08dd7c810}, 0x660089, {0xc014a46000, {0x174301104ef00b46, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc08dd7c810}, 0xc08b379500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104ef00b46, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104ef00b46, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08dd7c780}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08dd7c780}, {0xc002d353f8, 0x660089}, 0xc002d35400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08dd7c780}, {0x3fefffdc5f52ef1d}, 0xc08b379200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08dd7c6f0}, 0xc08b379200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}, 0xc08e0fa2e0, 0xc01636bc20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}, 0x3, 0xc08c99f540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002d35980}}}, {0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}, 0xc08e0fa2e0, 0xc08c99f540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08dd7c6f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08dd7c6f0}, {0x40, 0xc002d35b70}, 0xc002d35ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08dd7c6f0}, {0x4e9d7a0, 0xc08b379200}, 0xc08e0fa2e0, 0xc01636bc20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08dd7c6f0}, 0xc0305c5320, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0d22c5e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0d22c5e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804740 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc04d6fa060}, 0xc0193164f8, 0x1, 0xc086a952b0, 0x519bbc, 0xc01a184d80, 0xc073a67ac0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06f9690e0, {0x63c85b8, 0xc04d6fa060}, 0xc01a184de0, 0xc086a953b0, 0x18, 0x1e, 0x18, 0xc01a184d80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc04d6fa060}, 0xc01a184d20, {{{{0xc06b1150e0, 0xd}, {0xc01f393200, 0x18}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x1e, {0x63c85b8, 0xc04d6fa060}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc04d6fa060}, {0xc03e4859e0, {0x174301104c5c6ea6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc04d6fa060}, 0xc03b365f40, 0xc086a95bf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d6fa060}, 0x660089, {0xc03e4859e0, {0x174301104c5c6ea6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc04d6fa060}, 0xc04a417500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c5c6ea6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c5c6ea6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046f71e00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046f71e00}, {0xc086a97600, 0x660089}, 0xc086a97608) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046f71e00}, {0xc00102e4e0}, 0xc04a417400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046f71da0}, 0xc04a417400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8510, 0xc067947a40}, 0xe, 0xc04adc4680) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc0a1879a20, {0x63c8510, 0xc067947a40}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/txn.go:696 +0x74 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc00ae4a728, {0x63c8510, 0xc067947a40}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:225 +0xb6 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc00ae4a700, {{0x63c8510, 0xc067947a40}, 0xc059a95180, 0xc0d58639f0}) github.com/cockroachdb/cockroach/pkg/sql/update.go:196 +0x210 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc08a657278, {{0x63c8510, 0xc067947a40}, 0xc059a95180, 0xc0d58639f0}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc086a9b930, 0x63cb098}, {0x63caa08, 0xc08a657278}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:516 +0x147 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc086a9b930, {0x63caa08, 0xc08a657278}, {0x4ed688c, 0x5}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c9 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc086a9b930, {0x63caa08, 0xc08a657278}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x63c8510, 0xc067947a40}, 0xc059a95180, 0xc0d58639f0}, {0x63caa08, 0xc08a657278}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:519 +0x125 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc0588ce700, {0x63c8510, 0xc067947a40}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:146 +0x87 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:120 +0x34 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x200000000) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc063afbe60, {0x63c8510, 0xc067947a40}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:119 +0x6b github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc063afbe60, {0x63c8510, 0xc067947a40}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:753 +0x3a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc09d34b040, {0x63c8510, 0xc067947a40}, 0xc063afbe60) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:469 +0x258 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0d5863ce8, {0x63c8510, 0xc067947a40}, 0xc0a1879a20) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:303 +0x205 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc0679475c0}, 0xc0602f9dc0, 0xc0a1879a20, 0xc074c7bc80, 0xc0add20700, 0xc0d5863ce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc0679475c0}, 0xc0d58639f0, 0xc0602f9dc0, 0xc053a626c0, {{0x63caa08, 0xc08a657278}, 0x0}, 0xc0add20700) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0d5863300, {0x63c8510, 0xc0679475c0}, 0xc0d58639f0, 0xc0679475c0, {0x7fe42b790cf0, 0xc053a626c0}, 0x0, 0xc06d8ba898) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0d5863300, {0x63c8510, 0xc0679475c0}, 0xc0d58639f0, {0x7fe42b790cf0, 0xc053a626c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0d5863300, {0x63c85b8, 0xc066a74ae0}, {{0x63f9d78, 0xc018e2c700}, {0xc072b764e2, 0xa0}, 0x3, 0x1}, 0xc037d277a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0d5863300, {0x63c85b8, 0xc066a74ae0}, {{0x63f9d78, 0xc018e2c700}, {0xc072b764e2, 0xa0}, 0x3, 0x1}, 0xc037d277a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0d5863300, {0x63c85b8, 0xc066a74ae0}, {0xc037d277a0, {0xc070e0ee40, 0x3, 0x3}, {0x9bf64bc, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc072b7660c, 0x0}, 0x0, {0x31477ce4, 0xedb7a834f, 0x0}, 0x1}, 0xc0d5863300, 0xc086a9d980, 0xc086a9d970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0d5863300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0d5863300, {0x63c8510, 0xc0a83f51c0}, 0xc076677dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a4953d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0a83f51c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a4953d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808146 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc011762660}, 0xc028724f48, 0x1, 0xc00f0e50a8, 0x519bbc, 0xc07748bb60, 0xc0786a0ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc011762660}, 0x120, 0xc00f0e5230, 0x7748bb00, 0x0, 0x660069, 0xc07748bb60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc011762660}, 0xc07748bb00, {{{{0x0, 0x0}, {0xc059ed4d80, 0x11a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc011762660}, 0x6352410, 0x321a1798) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc011762660}, {0xc07748b9e0, {0x174301105a467403, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc011762660}, 0xc05b113360, 0xc00f0e59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc011762660}, 0x660089, {0xc07748b9e0, {0x174301105a467403, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc011762660}, 0xc032d10800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a467403, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a467403, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0117625d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0117625d0}, {0xc00f0e73f8, 0x660089}, 0xc00f0e7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0117625d0}, {0x7fe42870aaf8}, 0xc012d2f700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc011762540}, 0xc012d2f700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}, 0xc05f0728e0, 0xc021c01458) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}, 0x3, 0xc0479b9000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f0e7980}}}, {0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}, 0xc05f0728e0, 0xc0479b9000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc011762540}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc011762540}, {0x40, 0xc00f0e7b70}, 0xc00f0e7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc011762540}, {0x4e9d7a0, 0xc012d2f700}, 0xc05f0728e0, 0xc021c01458) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc011762540}, 0xc01f3d3500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07748b8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07748b8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809382 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc030bdf0b0}, 0xc0273487e0, 0x1, 0xc01a0af0a8, 0x519bbc, 0xc0c8e83a40, 0xc067429a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc030bdf0b0}, 0x120, 0xc01a0af230, 0xc8e839e0, 0x0, 0x660069, 0xc0c8e83a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc030bdf0b0}, 0xc0c8e839e0, {{{{0x0, 0x0}, {0xc0851a8fc0, 0xd2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc030bdf0b0}, 0x6352410, 0x300d0ad8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc030bdf0b0}, {0xc0c8e838c0, {0x174301105b86b204, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc030bdf0b0}, 0xc083c9f7c0, 0xc01a0af9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc030bdf0b0}, 0x660089, {0xc0c8e838c0, {0x174301105b86b204, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc030bdf0b0}, 0xc06ab86100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b86b204, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b86b204, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc030bdef30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030bdef30}, {0xc01a0b13f8, 0x660089}, 0xc01a0b1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc030bdef30}, {0x7fe3488954f8}, 0xc060ccbc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc030bdedb0}, 0xc060ccbc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}, 0xc0671ad640, 0xc03178fc50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}, 0x3, 0xc0b902b840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a0b1980}}}, {0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}, 0xc0671ad640, 0xc0b902b840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc030bdedb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030bdedb0}, {0x40, 0xc01a0b1b70}, 0xc01a0b1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc030bdedb0}, {0x4e9d7a0, 0xc060ccbc00}, 0xc0671ad640, 0xc03178fc50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc030bdedb0}, 0xc072e10720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0c8e837a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0c8e837a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812467 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc072bf0ab0}, 0xc06e3d6228, 0x1, 0xc03bf8b6d8, 0x519bbc, 0xc082444e40, 0xc076904300) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc030bbaf00, {0x63c85b8, 0xc072bf0ab0}, 0xc0aca897a0, 0xc03bf8b7d8, 0x82444e40, 0x0, 0xffffffffffffffff, 0xc082444e40, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc072bf0ab0}, 0xc082444d80, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc06de00000, 0xb1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc072bf0ab0}, 0x6352260, 0x33a92d50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc072bf0ab0}, {0xc082444c60, {0x1743011057a24815, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc072bf0ab0}, 0xc06891ab40, 0xc03bf8c020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc072bf0ab0}, 0x660089, {0xc082444c60, {0x1743011057a24815, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc072bf0ab0}, 0xc01a4e4200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057a24815, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072bf0a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072bf0a20}, {0xc07cdf5a28, 0x660089}, 0xc07cdf5a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072bf0a20}, {0x24}, 0xc01a4e4100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072bf0960}, 0xc01a4e4100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0197d7350, {0x63c85b8, 0xc072bf08a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0197d7350, {0x63c85b8, 0xc072bf08a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc072bf08a0}, {0x4f67ad2, 0x10beec0}, 0xc019549fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219806003 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc01c568270}, 0xc024f23710, 0x1, 0x0, 0x1, 0xc04ebe4600, 0xc08b6b3b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01c568270}, 0x120, 0xc005584728, 0x4ebe45a0, 0x0, 0x660069, 0xc04ebe4600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc01c568270}, 0xc04ebe45a0, {{{{0x0, 0x0}, {0xc09ce318c0, 0x59}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01c568270}, 0x6352410, 0x3a07eb00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc01c568270}, {0xc067337c20, {0x17430110499c7056, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc01c568270}, 0xc0845bc960, 0xc04c2f2ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01c568270}, 0x660089, {0xc067337c20, {0x17430110499c7056, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc01c568270}, 0xc04d55c200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110499c7056, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110499c7056, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c5681e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c5681e0}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c5681e0}, {0xc00102e4e0}, 0xc04d55c100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c568180}, 0xc04d55c100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03bc934a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791620 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc043afa8a0}, 0xc039a7bcb0, {0x646cc48, 0xc043afa900}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc043afa8a0}, 0xedb7a834e, 0xc087b57c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc043afa8a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0284ac5d0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc039a7bef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0284ac5d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812546 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc04f5ed8c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0d121ee00, 0xc0401dd168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04f5ed8c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0d121ee00, {0xc00ce1fa80, 0x29, 0x74}, {0xc04be8efc0, 0x29, 0x6d}, 0x0, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091747f38, {0xc00ce1fa80, 0x1, 0x6d}, {0xc04be8efc0, 0xc091746f00, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091747f38, {0xc00ce1fa80, 0x29, 0x74}, {0xc04be8efc0, 0x29, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0917471c0, {0xc04be8efc0, 0x0, 0xc0401dd4e0}, 0x121ea30, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0917471c0, {0xc00ce1fa80, 0x29, 0x6d}, {0xc04be8efc0, 0x28, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091746f00, {0xc04be8efc0, 0x29, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc086b84588, {{0xc03a9929f0, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07b309400, {{0xc03a9929f0, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04b6f3400, {0x63c85b8, 0xc04bf65bc0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04bf65bc0}, {0x64a6560, 0xc07b309400}, {0xc03a9929f0, 0x0, 0x30}, {0x17430110895b2e0a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04bf65bc0}, {0x6466ed8, 0xc086b84580}, {0xc03a9929f0, 0x28, 0x30}, {0x17430110895b2e0a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06daaf710}, {{0x17430110895b2e0a, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110895b2e0a, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc083616600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a728930a, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0312b0400, {0x63c85b8, 0xc04bf65bc0}, 0xc083616600, 0xc07076b400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0312b0400, {0x63c85b8, 0xc04bf65bc0}, 0xc083616600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110895b2e0a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110895b2e0a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bf65b30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf65b30}, {0xc0401e33f8, 0x660089}, 0xc0401e3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bf65b30}, {0xc0015221e0}, 0xc083616400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bf65aa0}, 0xc083616400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}, 0xc0ccf7f9a0, 0xc0381619c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}, 0x3, 0xc071e54500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0401e3980}}}, {0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}, 0xc0ccf7f9a0, 0xc071e54500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bf65aa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf65aa0}, {0x40, 0xc0401e3b70}, 0xc0401e3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bf65aa0}, {0x4e9d7a0, 0xc083616400}, 0xc0ccf7f9a0, 0xc0381619c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bf65aa0}, 0xc05c2fb980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812423 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0622c8850, {0x63c85b8, 0xc04f568f60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc04f568f60}, {0x4f67ad2, 0x10beec0}, 0xc02790dfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219802606 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021288000, {0x63c85b8, 0xc07b04ee10}, 0xc09fda8600, 0xc05307e460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021288000, {0x63c85b8, 0xc07b04ee10}, 0xc09fda8600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024a9bc59, 0x0, 0x0}, 0xc04fc36e20, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024a9bc59, 0x0, 0x0}, 0xc04fc36e20, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07b04ec60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b04ec60}, {0xc073095a08, 0x660089}, 0xc073095a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07b04ec60}, {0xc00102e4e0}, 0xc09fda8500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07b04ec00}, 0xc09fda8500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc04b90c750}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04b90c750}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799507 [runnable]: sync.runtime_SemacquireMutex(0xc094a56690, 0x8f, 0x7fe3348ba840) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc094a56538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc094a56560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc030330d80, {0xffffffffffffff15, 0xc094a566c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc030330d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc030330d80, {0xc021c12500, 0x32, 0x79}, {0xc02bc61280, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc078ce1298, {0xc021c12500, 0x32, 0x79}, {0xc02bc61280, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc078cdfbc0, {0xc02bc61280, 0x72, 0x79}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc078cdfbc0, {0xc021c12500, 0x72, 0xc02bc61280}, {0xc02bc61280, 0xc085188d20, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc078cdf900, {0xc02bc61280, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc052cec3a8, {{0xc085188d20, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04a1e7200, {{0xc047ee60f0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc048ba7c00, {0x63c85b8, 0xc077b97c80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc077b97c80}, {0x64a6560, 0xc04a1e7200}, {0xc047ee60f0, 0xedb79c942, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc077b97c80}, {0x6466ed8, 0xc052cec100}, {0xc047ee60f0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc044617f20}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03287c900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0205ca000, {0x63c85b8, 0xc077b97c80}, 0xc03287c900, 0xc0530a4e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205ca000, {0x63c85b8, 0xc077b97c80}, 0xc03287c900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077b97bc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077b97bc0}, {0xc0036f68f0, 0x660089}, 0xc0036f68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077b97bc0}, {0xc00102e4e0}, 0xc03287c800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077b97b60}, 0xc03287c800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc79080}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811911 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc02ee804e0}, 0xc0d3545308, 0x1, 0xc033d7b0a8, 0x519bbc, 0xc01610fda0, 0xc0589b6840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02ee804e0}, 0x120, 0xc033d7b230, 0x1610fd40, 0x0, 0x660069, 0xc01610fda0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc02ee804e0}, 0xc01610fd40, {{{{0x0, 0x0}, {0xc0457ba6c0, 0x33}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02ee804e0}, 0x6352410, 0x3c90c240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc02ee804e0}, {0xc01610fb00, {0x174301107847188e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc02ee804e0}, 0xc023e39360, 0xc033d7b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02ee804e0}, 0x660089, {0xc01610fb00, {0x174301107847188e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc02ee804e0}, 0xc082cc5600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107847188e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107847188e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ee80360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ee80360}, {0xc033d7d3f8, 0x660089}, 0xc033d7d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ee80360}, {0x0}, 0xc082cc5400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ee80210}, 0xc082cc5400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}, 0xc0bb7eb300, 0xc051b44d20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}, 0x3, 0xc076b1ba80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc033d7d980}}}, {0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}, 0xc0bb7eb300, 0xc076b1ba80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02ee80210}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ee80210}, {0x40, 0xc033d7db70}, 0xc033d7dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02ee80210}, {0x4e9d7a0, 0xc082cc5400}, 0xc0bb7eb300, 0xc051b44d20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02ee80210}, 0xc0563846c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc01610f8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc01610f8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806808 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b6f7540, {0x63c85b8, 0xc090ae2d50}, 0xc037122450, 0x1, 0xc004c2f0a8, 0x519bbc, 0xc00d7fe840, 0xc02edb5da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc090ae2d50}, 0x120, 0xc004c2f230, 0xd7fe7e0, 0x0, 0x660069, 0xc00d7fe840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b6f7540, {0x63c85b8, 0xc090ae2d50}, 0xc00d7fe7e0, {{{{0x0, 0x0}, {0xc023747200, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc090ae2d50}, 0x6352410, 0x45bf5750) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b6f4aa0, {0x63c85b8, 0xc090ae2d50}, {0xc00d7fe5a0, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b6f4aa0, {0x63c85b8, 0xc090ae2d50}, 0xc06877c8c0, 0xc004c2f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc090ae2d50}, 0x660089, {0xc00d7fe5a0, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b72bc00, {0x63c85b8, 0xc090ae2d50}, 0xc035fa8d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc090ae2cc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090ae2cc0}, {0xc004c313f8, 0x660089}, 0xc004c31400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc090ae2cc0}, {0x0}, 0xc035fa8b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc090ae2c30}, 0xc035fa8b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}, 0xc090a7f0c0, 0xc04b063788) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}, 0x3, 0xc08e687540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004c31980}}}, {0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}, 0xc090a7f0c0, 0xc08e687540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc090ae2c30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090ae2c30}, {0x40, 0xc004c31b70}, 0xc004c31ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc090ae2c30}, {0x4e9d7a0, 0xc035fa8b00}, 0xc090a7f0c0, 0xc04b063788) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc090ae2c30}, 0xc01b722a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00d7fe480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00d7fe480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806063 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc070c9cf00}, 0xc08a454ea0, 0x1, 0xc0075710a8, 0x519bbc, 0xc01bcdb500, 0xc05a1a7500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc070c9cf00}, 0x1e0, 0xc007571230, 0x1bcdb4a0, 0x0, 0x660069, 0xc01bcdb500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc070c9cf00}, 0xc01bcdb4a0, {{{{0x0, 0x0}, {0xc06f9690e0, 0x1a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc070c9cf00}, 0x63525f0, 0x1a9e3210) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc070c9cf00}, {0xc044ddc5a0, {0x174301104ba6f35b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc070c9cf00}, 0xc07c2f5cc0, 0xc0075719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc070c9cf00}, 0x660089, {0xc044ddc5a0, {0x174301104ba6f35b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc070c9cf00}, 0xc065022b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104ba6f35b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104ba6f35b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070c9ce70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070c9ce70}, {0xc0075733f8, 0x660089}, 0xc007573400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070c9ce70}, {0x2ea8494167a38a3b}, 0xc065022900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070c9cde0}, 0xc065022900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}, 0xc0797003e0, 0xc08a455e30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}, 0x3, 0xc062aaa740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007573980}}}, {0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}, 0xc0797003e0, 0xc062aaa740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070c9cde0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070c9cde0}, {0x40, 0xc007573b70}, 0xc007573ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070c9cde0}, {0x4e9d7a0, 0xc065022900}, 0xc0797003e0, 0xc08a455e30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070c9cde0}, 0xc046008de0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0784b7c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0784b7c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801374 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc025dbd400, {0x63c85b8, 0xc020272e70}, 0xc03668e700, 0xc0497f81e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025dbd400, {0x63c85b8, 0xc020272e70}, 0xc03668e700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c7e24b7, 0x0, 0x0}, 0xc06b486c90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c7e24b7, 0x0, 0x0}, 0xc06b486c90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020272de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020272de0}, {0xc04bc1f3f8, 0x660089}, 0xc04bc1f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020272de0}, {0x0}, 0xc06308de00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020272d20}, 0xc06308de00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}, 0xc00bdab1a0, 0xc042de2078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}, 0x3, 0xc0cedc0700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04bc1f980}}}, {0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}, 0xc00bdab1a0, 0xc0cedc0700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020272d20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020272d20}, {0x40, 0xc04bc1fb70}, 0xc04bc1fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020272d20}, {0x4e9d7a0, 0xc06308de00}, 0xc00bdab1a0, 0xc042de2078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020272d20}, 0xc04c68e420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ac167560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ac167560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795464 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa0698, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06b4d5180, 0xc00a18c800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06b4d5180, {0xc00a18c800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06b4d5180, {0xc00a18c800, 0x735, 0xc046d37cc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03a196000, {0xc00a18c800, 0x7eda79, 0xc00a18c805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc089ba8180, {0xc00a18c800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02f7533f8, {0xc00a18c800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc082ed1af8, {0x631b900, 0xc02f7533f8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc082ed1880, {0x7fe42ab59310, 0xc089ba8180}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc082ed1880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc082ed1880, {0xc0b63fc000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0614fd758, {0xc0b63fc000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc052ec2088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc052ec2088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc052ec2660, {0x634aac0, 0xc052ec2088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc052ec2000, 0xc07b27d2a0, 0xc044e0342b, 0xc044e0342d, {0x635bec0, 0xc012ff1b00}, 0xc000d6ae00, 0xc044533884, 0xc044e03429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc052ec2000, {0x63c85b8, 0xc06e24bb90}, 0xc07b27d290, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09f8b7038, 0x8}}, 0x0, 0xc06e24bb30, 0xc06e24bb60, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06e24bb90}, {0x643d810, 0xc089ba8180}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc062d70990}, {0x643d810, 0xc089ba8180}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc062d70990}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812580 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802461 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc02ecdc4b0}, {0xc061157ce0, 0x2, 0xc09c0c7e58}, {0x0, {0x1743010fecbb7edd, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc02ecdc4b0}, 0x18701, 0xc01d176ea0, 0x2b, 0xc053859670) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc02ecdc4b0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804953 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f6e8000, {0x63c85b8, 0xc05309a1b0}, 0xc0198aadc8, 0x1, 0xc08fd110a8, 0x519bbc, 0xc070306de0, 0xc06ddf6060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05309a1b0}, 0x120, 0xc08fd11230, 0x70306d80, 0xc06615b6c0, 0x18, 0xc070306de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f6e8000, {0x63c85b8, 0xc05309a1b0}, 0xc070306d80, {{{{0x0, 0x0}, {0xc03242e5a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05309a1b0}, 0x6352410, 0x32c54d10) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f6dc140, {0x63c85b8, 0xc05309a1b0}, {0xc070306b40, {0x174301103c0ea14d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f6dc140, {0x63c85b8, 0xc05309a1b0}, 0xc047208fa0, 0xc08fd119f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05309a1b0}, 0x660089, {0xc070306b40, {0x174301103c0ea14d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f6ea000, {0x63c85b8, 0xc05309a1b0}, 0xc06457aa00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103c0ea14d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103c0ea14d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05309a120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05309a120}, {0xc08fd133f8, 0x660089}, 0xc08fd13400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05309a120}, {0x70}, 0xc06457a800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05309a090}, 0xc06457a800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}, 0xc06649a9e0, 0xc031050d08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}, 0x3, 0xc05969b340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc08fd13980}}}, {0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}, 0xc06649a9e0, 0xc05969b340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05309a090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05309a090}, {0x40, 0xc08fd13b70}, 0xc08fd13ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}, 0x203016, 0x203016) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05309a090}, {0x4e9d7a0, 0xc06457a800}, 0xc06649a9e0, 0xc031050d08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05309a090}, 0xc031304f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc070306a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc070306a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806708 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021d20400, {0x63c85b8, 0xc08dfd0ff0}, 0xc08539dd00, 0xc02b3e06e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021d20400, {0x63c85b8, 0xc08dfd0ff0}, 0xc08539dd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104ec6aed2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104ec6aed2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08dfd0f60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08dfd0f60}, {0xc002a1f3f8, 0x660089}, 0xc002a1f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08dfd0f60}, {0x0}, 0xc08539db00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08dfd0ed0}, 0xc08539db00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}, 0xc09069c5a0, 0xc01f9b71a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}, 0x3, 0xc08ba419c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002a1f980}}}, {0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}, 0xc09069c5a0, 0xc08ba419c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08dfd0ed0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08dfd0ed0}, {0x40, 0xc002a1fb70}, 0xc002a1fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08dfd0ed0}, {0x4e9d7a0, 0xc08539db00}, 0xc09069c5a0, 0xc01f9b71a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08dfd0ed0}, 0xc0b7cde240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00d168120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00d168120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806497 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc08dfd00f0}, 0xc01f9b6ae0, 0x1, 0x708017ecd63ca7, 0x37e11d600, 0xc04c0fa060, 0xc06bf6f320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc08dfd00f0}, 0x1e0, 0xc003311230, 0x4c0fa000, 0x0, 0x660069, 0xc04c0fa060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc08dfd00f0}, 0xc04c0fa000, {{{{0x0, 0x0}, {0xc093c1f8c0, 0x10d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc0c87eb500, {0x63c85b8, 0xc08dfd00f0}, 0xc04be56610, 0x3311428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc08dfd00f0}, {0xc0138f4a20, {0x174301104e7fca81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc08dfd00f0}, 0xc02b3e01e0, 0xc0033119f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08dfd00f0}, 0x660089, {0xc0138f4a20, {0x174301104e7fca81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc08dfd00f0}, 0xc08539d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e7fca81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e7fca81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08dfd0060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08dfd0060}, {0xc0033133f8, 0x660089}, 0xc003313400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08dfd0060}, {0x3ea070552fe6acc2}, 0xc08539d200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08d4b1fb0}, 0xc08539d200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}, 0xc09069c0a0, 0xc01f9b6ab0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}, 0x3, 0xc08ba41100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003313980}}}, {0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}, 0xc09069c0a0, 0xc08ba41100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08d4b1fb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d4b1fb0}, {0x40, 0xc003313b70}, 0xc003313ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08d4b1fb0}, {0x4e9d7a0, 0xc08539d200}, 0xc09069c0a0, 0xc01f9b6ab0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08d4b1fb0}, 0xc0bd4b9440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0138f45a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0138f45a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805643 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0185643c0, {0x63c85b8, 0xc0439732c0}, 0xc015cd1a28, 0x1, 0xc00565b0a8, 0x519bbc, 0xc0c7dc1f00, 0xc06ec231a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc040092480, {0x63c85b8, 0xc0439732c0}, 0xc0c7dc1f00, 0xc00565b1a8, 0x30, 0x40, 0x4, 0xc0c7dc1f00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0185643c0, {0x63c85b8, 0xc0439732c0}, 0xc0499e1d40, {{{{0xc05a4837a0, 0x2}, {0xc0427445a0, 0x2}}, {{0xc083495440, 0x8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0439732c0}, 0x63521a0, 0x1cdd9680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0185322d0, {0x63c85b8, 0xc0439732c0}, {0xc04e46b8c0, {0x1743010fff22114f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0185322d0, {0x63c85b8, 0xc0439732c0}, 0xc02d431720, 0xc00565b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0439732c0}, 0x660089, {0xc04e46b8c0, {0x1743010fff22114f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018570c00, {0x63c85b8, 0xc0439732c0}, 0xc063448400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fff22114f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fff22114f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043973200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043973200}, {0xc00565d3f8, 0x660089}, 0xc00565d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043973200}, {0xc055efe500}, 0xc063448100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043973140}, 0xc063448100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}, 0xc06deb8520, 0xc07d344b40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}, 0x3, 0xc092d4bf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00565d980}}}, {0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}, 0xc06deb8520, 0xc092d4bf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc043973140}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043973140}, {0x40, 0xc00565db70}, 0xc00565dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc043973140}, {0x4e9d7a0, 0xc063448100}, 0xc06deb8520, 0xc07d344b40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc043973140}, 0xc0499e1bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04e46b560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04e46b560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812190 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc025026280, {0x63c85b8, 0xc0737766c0}, 0xc0204ccbb8, 0x1, 0xc03994f0a8, 0x519bbc, 0xc013f31800, 0xc019fdf6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0737766c0}, 0x120, 0xc03994f230, 0x13f317a0, 0x0, 0x660069, 0xc013f31800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc025026280, {0x63c85b8, 0xc0737766c0}, 0xc013f317a0, {{{{0x0, 0x0}, {0xc0765b3200, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0737766c0}, 0x6352410, 0x17838998) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0234e8a50, {0x63c85b8, 0xc0737766c0}, {0xc013f31680, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0234e8a50, {0x63c85b8, 0xc0737766c0}, 0xc0423419a0, 0xc03994f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0737766c0}, 0x660089, {0xc013f31680, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025034000, {0x63c85b8, 0xc0737766c0}, 0xc06d230d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0737765d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0737765d0}, {0xc0399513f8, 0x660089}, 0xc039951400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0737765d0}, {0x7fe3c2fa3188}, 0xc06d230b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073776540}, 0xc06d230b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}, 0xc0486a78a0, 0xc0204cd158) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}, 0x3, 0xc05750b3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc039951980}}}, {0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}, 0xc0486a78a0, 0xc05750b3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073776540}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073776540}, {0x40, 0xc039951b70}, 0xc039951ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073776540}, {0x4e9d7a0, 0xc06d230b00}, 0xc0486a78a0, 0xc0204cd158) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073776540}, 0xc02d783ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f31560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f31560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810007 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc051603e60}, 0xc0affa9d88, 0x1, 0xc005ddb670, 0x519bbc, 0xc04efead80, 0xc0730ddaa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06e335b00, {0x63c85b8, 0xc051603e60}, 0xc04166b440, 0xc005ddb770, 0x1c97ccf, 0xc07cd11560, 0x0, 0xc04efead80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc051603e60}, 0xc0395e30e0, {{{{0xc079db7b00, 0x46}, {0xc04bfde000, 0x6a}}, {{0xc035cd66c0, 0x94}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc051603e60}, 0x63521a0, 0x1fa2f778) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc051603e60}, {0xc03fc707e0, {0x1743011002b22575, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc051603e60}, 0xc02c4b01e0, 0xc005ddbfb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc051603e60}, 0x660089, {0xc03fc707e0, {0x1743011002b22575, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc051603e60}, 0xc057b94900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011002b22575, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011002b22575, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc051603dd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051603dd0}, {0xc07ce539c0, 0x660089}, 0xc07ce539c8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc051603dd0}, {0xc00102e4e0}, 0xc057b94800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc051603d70}, 0xc057b94800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x63c85b8, 0xc0516037d0}, {0x7fe422a51230, 0xc07d646500}, 0xc052e20b40, {0xc03353be60, 0x6, 0x6}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x24d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x63c85b8, 0xc0516037d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 214514382 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc039c2d050, 0x2d0) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc026351500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc026351b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0bc710600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0bc710600, {0x63c8510, 0xc039c2d080}, 0xc057611dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0134b87a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc039c2d080}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0134b87a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219800772 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc096864548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0968645b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc090e036f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc081bcf180, {0xc015766380, 0x2b, 0x6e}, {0xc011a23ef0, 0x2b, 0x84}, 0x1, 0x36) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc090e03588, {0xc015766380, 0x1, 0x84}, {0xc011a23ef0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc090e03588, {0xc015766380, 0x2b, 0x6e}, {0xc011a23ef0, 0x2b, 0x84}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc090e02cc0, {0xc011a23ef0, 0x84, 0x6e}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc090e02cc0, {0xc015766380, 0x2b, 0xc011a23ef0}, {0xc011a23ef0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc090e02a00, {0xc011a23ef0, 0x2b, 0x84}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0c186eb08, {{0xc05379b470, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04eeb8400, {{0xc05379b470, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ef74800, {0x63c85b8, 0xc05c639230}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc05c639230}, {0x64a6560, 0xc04eeb8400}, {0xc05379b470, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc05c639230}, {0x6466ed8, 0xc0c186eb00}, {0xc05379b470, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc03067b110}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03cbb5b00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b13400, {0x63c85b8, 0xc05c639230}, 0xc03cbb5b00, 0xc03953f900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b13400, {0x63c85b8, 0xc05c639230}, 0xc03cbb5b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05c639140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c639140}, {0xc0936948f0, 0x660089}, 0xc0936948f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05c639140}, {0xc00102e4e0}, 0xc03cbb5a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05c6390e0}, 0xc03cbb5a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02aa62180}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803994 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc029a89710}, 0xc07289a8d0, 0x1, 0xc075f950a8, 0x519bbc, 0xc04c2b5c80, 0xc04d952600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc029a89710}, 0x1e0, 0xc075f95230, 0x4c2b5c20, 0x0, 0x660069, 0xc04c2b5c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc029a89710}, 0xc04c2b5c20, {{{{0x0, 0x0}, {0xc04514afc0, 0x50}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc075f95440, {0x63c85b8, 0xc029a89710}, 0xc049ea0000, 0x75f95428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc029a89710}, {0xc050d4fc20, {0x174301103094392a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc029a89710}, 0xc0b16d7ea0, 0xc075f959f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc029a89710}, 0x660089, {0xc050d4fc20, {0x174301103094392a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc029a89710}, 0xc03b047200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029a89620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029a89620}, {0xc075f973f8, 0x660089}, 0xc075f97400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029a89620}, {0xc08e0a27e0}, 0xc03b046f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029a89560}, 0xc03b046f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}, 0xc03ef819e0, 0xc041389bd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}, 0x3, 0xc0180d5340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075f97980}}}, {0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}, 0xc03ef819e0, 0xc0180d5340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029a89560}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029a89560}, {0x40, 0xc075f97b70}, 0xc075f97ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029a89560}, {0x4e9d7a0, 0xc03b046f00}, 0xc03ef819e0, 0xc041389bd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029a89560}, 0xc0425a34a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050d4fb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050d4fb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810030 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018112500, {0x63c85b8, 0xc01e078d50}, 0xc04a1c9860, 0x1, 0xc0077250a8, 0x519bbc, 0xc06e413f00, 0xc03604a780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc073caaa20, {0x63c85b8, 0xc01e078d50}, 0xc06e413f00, 0xc0077251a8, 0x30, 0x40, 0x0, 0xc06e413f00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018112500, {0x63c85b8, 0xc01e078d50}, 0xc00e02de00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0d6108360, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01e078d50}, 0x63521a0, 0x4b1bd690) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc017e6f450, {0x63c85b8, 0xc01e078d50}, {0xc066c1b440, {0x1743010fef6c6cfa, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc017e6f450, {0x63c85b8, 0xc01e078d50}, 0xc062ed3a40, 0xc0077259f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01e078d50}, 0x660089, {0xc066c1b440, {0x1743010fef6c6cfa, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01811f400, {0x63c85b8, 0xc01e078d50}, 0xc074122100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fef6c6cfa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fef6c6cfa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e078c60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e078c60}, {0xc0077273f8, 0x660089}, 0xc007727400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e078c60}, {0x7fe13dc44138}, 0xc048877d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e078b40}, 0xc048877d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}, 0xc078f0e300, 0xc072e1e9f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}, 0x3, 0xc0293ced40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007727980}}}, {0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}, 0xc078f0e300, 0xc0293ced40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01e078b40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e078b40}, {0x40, 0xc007727b70}, 0xc007727ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01e078b40}, {0x4e9d7a0, 0xc048877d00}, 0xc078f0e300, 0xc072e1e9f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01e078b40}, 0xc00e02dbc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01e819320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01e819320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810281 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02532b400, {0x63c85b8, 0xc04a294810}, 0xc089187b00, 0xc02bd9c140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc04a294810}, 0xc089187b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106ca36aae, 0x0, 0x0}, 0xc0649f1b00, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106ca36aae, 0x0, 0x0}, 0xc0649f1b00, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04a294750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a294750}, {0xc01a0093f8, 0x660089}, 0xc01a009400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04a294750}, {0x7fe3c2f0ad80}, 0xc089187900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04a294660}, 0xc089187900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}, 0xc0301e4b60, 0xc0361348e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}, 0x3, 0xc08a21d4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a009980}}}, {0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}, 0xc0301e4b60, 0xc08a21d4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04a294660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a294660}, {0x40, 0xc01a009b70}, 0xc01a009ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04a294660}, {0x4e9d7a0, 0xc089187900}, 0xc0301e4b60, 0xc0361348e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04a294660}, 0xc0b48959e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc016409680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc016409680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804907 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030b12000, {0x63c85b8, 0xc07282eb40}, 0xc066aad700, 0xc03e4bce60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030b12000, {0x63c85b8, 0xc07282eb40}, 0xc066aad700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011035513431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011035513431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07282eab0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07282eab0}, {0xc094f933f8, 0x660089}, 0xc094f93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07282eab0}, {0x7fe3c2f5f870}, 0xc066aad500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07282ea20}, 0xc066aad500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}, 0xc078d0d1a0, 0xc0594b6300) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}, 0x3, 0xc042838b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc094f93980}}}, {0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}, 0xc078d0d1a0, 0xc042838b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07282ea20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07282ea20}, {0x40, 0xc094f93b70}, 0xc094f93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}, 0x203010, 0x203010) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07282ea20}, {0x4e9d7a0, 0xc066aad500}, 0xc078d0d1a0, 0xc0594b6300) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07282ea20}, 0xc0bcbac480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058c1fb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058c1fb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803401 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc06ffeec00, {0x63c85b8, 0xc055b6f860}, 0xc065724900, 0xc04c4495e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc06ffeec00, {0x63c85b8, 0xc055b6f860}, 0xc065724900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bb82fa6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bb82fa6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc055b6f7d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc055b6f7d0}, {0xc05579b3f8, 0x660089}, 0xc05579b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc055b6f7d0}, {0xc08bd1e870}, 0xc065724700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc055b6f6b0}, 0xc065724700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}, 0xc0559198e0, 0xc0453283c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}, 0x3, 0xc0b0b23ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05579b980}}}, {0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}, 0xc0559198e0, 0xc0b0b23ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc055b6f6b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc055b6f6b0}, {0x40, 0xc05579bb70}, 0xc05579bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc055b6f6b0}, {0x4e9d7a0, 0xc065724700}, 0xc0559198e0, 0xc0453283c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc055b6f6b0}, 0xc06e67b3e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc037e239e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc037e239e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809461 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc077a15980}, 0xc03ef88c30, 0x1, 0xc01e8d10a8, 0x519bbc, 0xc0b11a2de0, 0xc052db8840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077a15980}, 0x120, 0xc01e8d1230, 0xb11a2d80, 0x0, 0x660069, 0xc0b11a2de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc077a15980}, 0xc0b11a2d80, {{{{0x0, 0x0}, {0xc03e7946c0, 0xd4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077a15980}, 0x6352410, 0x450cb208) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc077a15980}, {0xc0b11a2c60, {0x174301106376d365, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc077a15980}, 0xc0225cf7c0, 0xc01e8d19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077a15980}, 0x660089, {0xc0b11a2c60, {0x174301106376d365, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc077a15980}, 0xc00c027b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106376d365, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106376d365, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077a158f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077a158f0}, {0xc01e8d33f8, 0x660089}, 0xc01e8d3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077a158f0}, {0x3feffffe539c41e7}, 0xc00c027700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077a15830}, 0xc00c027700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}, 0xc034a63040, 0xc086c7b9c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}, 0x3, 0xc03951e400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01e8d3980}}}, {0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}, 0xc034a63040, 0xc03951e400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc077a15830}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077a15830}, {0x40, 0xc01e8d3b70}, 0xc01e8d3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc077a15830}, {0x4e9d7a0, 0xc00c027700}, 0xc034a63040, 0xc086c7b9c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc077a15830}, 0xc04d8ae1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0b11a2b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0b11a2b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802328 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01bd13400, {0x63c85b8, 0xc075752930}, 0xc049d87c00, 0xc0589d2d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01bd13400, {0x63c85b8, 0xc075752930}, 0xc049d87c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011021de0ead, 0x0, 0x0}, 0xc024e29f60, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011021de0ead, 0x0, 0x0}, 0xc024e29f60, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0757528a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0757528a0}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0757528a0}, {0xc00102e4e0}, 0xc049d87a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc075752840}, 0xc049d87a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc04117f740}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0757527b0}, {0xc00dfd36b0, 0x24}, 0x46c773f8c, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc02df95dd0, 0xc001afcc78}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0757527b0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219794640 [runnable]: sync.runtime_SemacquireMutex(0x4, 0x60, 0x2a) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0315303c0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc09171fbf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc09fa3ea80, {0xc00c370ee0, 0x32, 0x6d}, {0xc02d4cec80, 0x32, 0x74}, 0x1, 0xfb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc09171fa88, {0xc00c370ee0, 0x1, 0x74}, {0xc02d4cec80, 0x2aa, 0xc006059298}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09171fa88, {0xc00c370ee0, 0x32, 0x6d}, {0xc02d4cec80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09171f1c0, {0xc02d4cec80, 0x74, 0x6d}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09171f1c0, {0xc00c370ee0, 0x74, 0xc02d4cec80}, {0xc02d4cec80, 0xc0931e3d40, 0x31}, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09171ef00, {0xc02d4cec80, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0af74c928, {{0xc0931e3d40, 0x31, 0xc0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04830a600, {{0xc083e51890, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0cc0cb800, {0x63c85b8, 0xc02f28e630}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02f28e630}, {0x64a6560, 0xc04830a600}, {0xc083e51890, 0xc006059e68, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02f28e630}, {0x6466ed8, 0xc0af74c680}, {0xc083e51890, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0654a7e48}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04c3b1300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b0ec00, {0x63c85b8, 0xc02f28e630}, 0xc04c3b1300, 0xc052cf0e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b0ec00, {0x63c85b8, 0xc02f28e630}, 0xc04c3b1300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f28e510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f28e510}, {0xc00605f3f8, 0x660089}, 0xc00605f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f28e510}, {0xc04b66e280}, 0xc04c3b1000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f28e300}, 0xc04c3b1000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}, 0xc07c7d5cc0, 0xc045f1e798) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}, 0x3, 0xc020e29b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00605f980}}}, {0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}, 0xc07c7d5cc0, 0xc020e29b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f28e300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f28e300}, {0x40, 0xc00605fb70}, 0xc00605fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f28e300}, {0x4e9d7a0, 0xc04c3b1000}, 0xc07c7d5cc0, 0xc045f1e798) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f28e300}, 0xc059ec6f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0153c6000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0153c6000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806491 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc08b8c00f0}, 0xc0909dc5d0, 0x1, 0xc00def90a8, 0x519bbc, 0xc02c4d94a0, 0xc059c7cf00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08b8c00f0}, 0x120, 0xc00def9230, 0x2c4d9440, 0x0, 0x660069, 0xc02c4d94a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc08b8c00f0}, 0xc02c4d9440, {{{{0x0, 0x0}, {0xc07227f680, 0x24}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08b8c00f0}, 0x6352410, 0x23d187e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc08b8c00f0}, {0xc02c4d9320, {0x174301104e81a176, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc08b8c00f0}, 0xc06ac586e0, 0xc00def99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08b8c00f0}, 0x660089, {0xc02c4d9320, {0x174301104e81a176, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc08b8c00f0}, 0xc01fb73600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e81a176, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e81a176, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08b8c0060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08b8c0060}, {0xc00defb3f8, 0x660089}, 0xc00defb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08b8c0060}, {0x0}, 0xc01fb72a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08a19dfb0}, 0xc01fb72a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}, 0xc08b880e00, 0xc02866a9f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}, 0x3, 0xc07684ee80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00defb980}}}, {0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}, 0xc08b880e00, 0xc07684ee80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08a19dfb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a19dfb0}, {0x40, 0xc00defbb70}, 0xc00defbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08a19dfb0}, {0x4e9d7a0, 0xc01fb72a00}, 0xc08b880e00, 0xc02866a9f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08a19dfb0}, 0xc04ebf3440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02c4d9200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02c4d9200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812677 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch.func4.1({0x63c8580, 0xc03df197a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:301 +0xf6 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02557bcb0}, {0x4f0948f, 0x11}, 0x4a817c800, 0xc034637ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch.func4({0x63c85b8, 0xc02557bcb0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:299 +0x91 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805342 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024faf680, {0x63c85b8, 0xc04f1a7650}, 0xc015219278, 0x1, 0x708017dfd7c367, 0x37e11d600, 0xc05b3cd4a0, 0xc067945140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04f1a7650}, 0x120, 0xc087ad7230, 0x5b3cd440, 0xc0722e1ba0, 0x18, 0xc05b3cd4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024faf680, {0x63c85b8, 0xc04f1a7650}, 0xc05b3cd440, {{{{0x0, 0x0}, {0xc04fbe90e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f1a7650}, 0x6352410, 0x24ff0fc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024ef1d60, {0x63c85b8, 0xc04f1a7650}, {0xc05b3cd320, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024ef1d60, {0x63c85b8, 0xc04f1a7650}, 0xc049ba9a40, 0xc087ad79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f1a7650}, 0x660089, {0xc05b3cd320, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025097800, {0x63c85b8, 0xc04f1a7650}, 0xc052a72000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f1a7530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f1a7530}, {0xc087ad93f8, 0x660089}, 0xc087ad9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f1a7530}, {0xc0015221e0}, 0xc052c3de00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f1a74a0}, 0xc052c3de00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}, 0xc0656ebe40, 0xc015219248) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}, 0x3, 0xc06c770cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc087ad9980}}}, {0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}, 0xc0656ebe40, 0xc06c770cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f1a74a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f1a74a0}, {0x40, 0xc087ad9b70}, 0xc087ad9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f1a74a0}, {0x4e9d7a0, 0xc052c3de00}, 0xc0656ebe40, 0xc015219248) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f1a74a0}, 0xc0ce586fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b776000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b776000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812545 [runnable]: sync.runtime_SemacquireMutex(0x7fe3aaf1c300, 0xc8, 0x459e98) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x1194a0b, 0x7fd57e391480, 0x1e2000f) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc03ff1cef0, 0x4d92f8, 0xc03ff1cf28, 0x119e525) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00def22c0, {0x1c2f55, 0x4455}, 0x0, 0xc074f87778) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc074f87500, {0xffffffffffffff15, 0xc03ff1d0a0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc074f87500) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc074f87500, {0xc08fcbf240, 0x17, 0x31}, 0x8, 0xa8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekGE(0xc092e80640, {0xc08fcbf240, 0x1, 0x31}, 0x33) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:513 +0xaf github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc092e80640, {0xc08fcbf240, 0x17, 0x31}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:409 +0xd6 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092e7f1c0, {0xc08fcbf240, 0x30, 0x31}, 0x17, 0x17) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc092e7f1c0, {0xc08fcbf240, 0xc08fcbf240, 0xc06d2eec90}, 0x31) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc092e7ef00, {0xc08fcbf240, 0x17, 0x31}, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/pebble.(*Iterator).SeekGE(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:865 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc09abffcd8, {{0xc0578ee450, 0x16, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:290 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07376fb00, {{0xc0578ee450, 0x16, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc083e5c400, {{0xc0578ee450, 0x16, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc083e5c400, {0x63c85b8, 0xc04bf659e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc0578ee468, 0x16, 0x18}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.MVCCScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc0578ee468, 0x16, 0x18}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2504 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc0d233a198}, {{0x17430110883d5b6b, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:62 +0x41b github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110883d5b6b, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc083616300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a60ac06b, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e749800, {0x63c85b8, 0xc04bf659e0}, 0xc083616300, 0xc0777fe0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e749800, {0x63c85b8, 0xc04bf659e0}, 0xc083616300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110883d5b6b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110883d5b6b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bf65950}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf65950}, {0xc03ff233f8, 0x660089}, 0xc03ff23400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bf65950}, {0x4044fffcb3216cf9}, 0xc083616000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bf658c0}, 0xc083616000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}, 0xc0ccf7f940, 0xc038161950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}, 0x3, 0xc071e54380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03ff23980}}}, {0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}, 0xc0ccf7f940, 0xc071e54380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bf658c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf658c0}, {0x40, 0xc03ff23b70}, 0xc03ff23ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bf658c0}, {0x4e9d7a0, 0xc083616000}, 0xc0ccf7f940, 0xc038161950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bf658c0}, 0xc05c2fb680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812318 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0673d3d50, {0x63c85b8, 0xc0707b4090}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0707b4090}, {0x4f67ad2, 0x10beec0}, 0xc002d9cfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219809460 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01f04d900, {0x63c85b8, 0xc0256343c0}, 0xc016a612d8, 0x1, 0xc00a4750a8, 0x519bbc, 0xc05aa1fa40, 0xc0524fdb60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0256343c0}, 0x120, 0xc00a475230, 0x5aa1f9e0, 0x0, 0x660069, 0xc05aa1fa40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01f04d900, {0x63c85b8, 0xc0256343c0}, 0xc05aa1f9e0, {{{{0x0, 0x0}, {0xc0d6109c20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0256343c0}, 0x6352410, 0x46e77460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01ef6db80, {0x63c85b8, 0xc0256343c0}, {0xc05aa1f8c0, {0x174301106376d365, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01ef6db80, {0x63c85b8, 0xc0256343c0}, 0xc083619f40, 0xc00a4759f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0256343c0}, 0x660089, {0xc05aa1f8c0, {0x174301106376d365, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f18cc00, {0x63c85b8, 0xc0256343c0}, 0xc05f8cec00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106376d365, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106376d365, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0256341b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0256341b0}, {0xc00a4773f8, 0x660089}, 0xc00a477400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0256341b0}, {0x7fe3b7a9cc30}, 0xc05f8ce900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021383b60}, 0xc05f8ce900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}, 0xc047482d40, 0xc01ea1d4e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}, 0x3, 0xc064de8180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a477980}}}, {0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}, 0xc047482d40, 0xc064de8180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc021383b60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021383b60}, {0x40, 0xc00a477b70}, 0xc00a477ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc021383b60}, {0x4e9d7a0, 0xc05f8ce900}, 0xc047482d40, 0xc01ea1d4e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc021383b60}, 0xc02fd00f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0b11a2a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0b11a2a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810373 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc040e7d770}, 0xc06d66c828, 0x1, 0xc0272a16d8, 0x519bbc, 0xc01fa53bc0, 0xc06bb7b080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc084e4aae0, {0x63c85b8, 0xc040e7d770}, 0xc067749e60, 0xc0272a17d8, 0x1fa53bc0, 0x0, 0x0, 0xc01fa53bc0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc040e7d770}, 0xc01fa53b00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc077b05440, 0xa2}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc040e7d770}, 0x6352260, 0x51b88990) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc040e7d770}, {0xc01fa537a0, {0x17430110310ddab1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc040e7d770}, 0xc0676e1ea0, 0xc0272a2020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc040e7d770}, 0x660089, {0xc01fa537a0, {0x17430110310ddab1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc040e7d770}, 0xc088381200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc040e7d6e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc040e7d6e0}, {0xc081e5da28, 0x660089}, 0xc081e5da30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc040e7d6e0}, {0xc00102e4e0}, 0xc088381100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc040e7d680}, 0xc088381100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc050d95d50, {0x63c85b8, 0xc040e7d5c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc050d95d50, {0x63c85b8, 0xc040e7d5c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc040e7d5c0}, {0x4f67ad2, 0x6446ff8}, 0xc012270fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219803100 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0213bbe00, {0x63c85b8, 0xc06489bfb0}, 0xc06d430048, 0x1, 0xc05390f0a8, 0x519bbc, 0xc062f016e0, 0xc05faf22a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06489bfb0}, 0x120, 0xc05390f230, 0x62f01680, 0x0, 0x660069, 0xc062f016e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0213bbe00, {0x63c85b8, 0xc06489bfb0}, 0xc062f01680, {{{{0x0, 0x0}, {0xc0d6108c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06489bfb0}, 0x6352410, 0x49a43248) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135ba90, {0x63c85b8, 0xc06489bfb0}, {0xc062f01560, {0x1743011026aefceb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135ba90, {0x63c85b8, 0xc06489bfb0}, 0xc02eb38e60, 0xc05390f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06489bfb0}, 0x660089, {0xc062f01560, {0x1743011026aefceb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214a9400, {0x63c85b8, 0xc06489bfb0}, 0xc03b7cfb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011026aefceb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011026aefceb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06489bef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06489bef0}, {0xc0539113f8, 0x660089}, 0xc053911400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06489bef0}, {0xc0191256c0}, 0xc03b7cf900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06489be60}, 0xc03b7cf900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}, 0xc072c82f80, 0xc028a606f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}, 0x3, 0xc02846d8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc053911980}}}, {0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}, 0xc072c82f80, 0xc02846d8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06489be60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06489be60}, {0x40, 0xc053911b70}, 0xc053911ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06489be60}, {0x4e9d7a0, 0xc03b7cf900}, 0xc072c82f80, 0xc028a606f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06489be60}, 0xc02418e720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc062f01440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc062f01440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812551 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc027204100, 0xc096804e00, 0xc04146f168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc096804e00, {0xc02e00f7a0, 0x30, 0x52}, {0xc0580233e0, 0x30, 0x30}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0915fff88, {0xc02e00f7a0, 0x1, 0x30}, {0xc0580233e0, 0xc0915ff400, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0915fff88, {0xc02e00f7a0, 0x30, 0x52}, {0xc0580233e0, 0x30, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0915ff6c0, {0xc0580233e0, 0x0, 0xc04146f4e0}, 0x121ea30, 0x50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0915ff6c0, {0xc02e00f7a0, 0x30, 0x30}, {0xc0580233e0, 0x2f, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0915ff400, {0xc0580233e0, 0x30, 0x30}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc041fbcb08, {{0xc03a992bd0, 0x2f, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc09a39b400, {{0xc03a992bd0, 0x2f, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0b72f7c00, {0x63c85b8, 0xc0566145a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0566145a0}, {0x64a6560, 0xc09a39b400}, {0xc03a992bd0, 0xc00bc1fa40, 0x30}, {0x174301108b396d7b, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0566145a0}, {0x6466ed8, 0xc041fbcb00}, {0xc03a992bd0, 0x2f, 0x30}, {0x174301108b396d7b, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc046ebe2e8}, {{0x174301108b396d7b, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301108b396d7b, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc083617300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a906d27b, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0219b8000, {0x63c85b8, 0xc0566145a0}, 0xc083617300, 0xc0327697c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc0566145a0}, 0xc083617300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108b396d7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108b396d7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056614510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614510}, {0xc0414753f8, 0x660089}, 0xc041475400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056614510}, {0x80a2a8801003d3d}, 0xc083617100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056614480}, 0xc083617100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}, 0xc0ccf7fbc0, 0xc038161c38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}, 0x3, 0xc071e54980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc041475980}}}, {0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}, 0xc0ccf7fbc0, 0xc071e54980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056614480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614480}, {0x40, 0xc041475b70}, 0xc041475ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056614480}, {0x4e9d7a0, 0xc083617100}, 0xc0ccf7fbc0, 0xc038161c38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056614480}, 0xc071982540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812651 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806698 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03100c780, {0x63c85b8, 0xc077c11050}, 0xc02febea68, 0x1, 0xc0032930a8, 0x519bbc, 0xc003c15880, 0xc06bb7b860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc057368c60, {0x63c85b8, 0xc077c11050}, 0xc003c15880, 0xc0032931a8, 0x42, 0x60, 0x0, 0xc003c15880, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03100c780, {0x63c85b8, 0xc077c11050}, 0xc03b95f380, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc067752000, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077c11050}, 0x63521a0, 0x3a07f628) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030f7fbd0, {0x63c85b8, 0xc077c11050}, {0xc0777dfc20, {0x1743010ffec7dd4e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030f7fbd0, {0x63c85b8, 0xc077c11050}, 0xc054ef6320, 0xc0032939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077c11050}, 0x660089, {0xc0777dfc20, {0x1743010ffec7dd4e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc03100b400, {0x63c85b8, 0xc077c11050}, 0xc08fb7a000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffec7dd4e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffec7dd4e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077c10fc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077c10fc0}, {0xc0032953f8, 0x660089}, 0xc003295400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077c10fc0}, {0x7fe42863f220}, 0xc089079d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077c10f00}, 0xc089079d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}, 0xc08ef7d700, 0xc044f74ac8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}, 0x3, 0xc086e3e700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003295980}}}, {0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}, 0xc08ef7d700, 0xc086e3e700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc077c10f00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077c10f00}, {0x40, 0xc003295b70}, 0xc003295ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc077c10f00}, {0x4e9d7a0, 0xc089079d00}, 0xc08ef7d700, 0xc044f74ac8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc077c10f00}, 0xc03b95f200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0777dfb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0777dfb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810732 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0716f9400, {0x63c85b8, 0xc06e8d61e0}, 0xc08ad97300, 0xc07920c960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0716f9400, {0x63c85b8, 0xc06e8d61e0}, 0xc08ad97300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106e46f97c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106e46f97c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e8d6150}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e8d6150}, {0xc013b973f8, 0x660089}, 0xc013b97400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e8d6150}, {0xf4}, 0xc08ad97100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e8d6090}, 0xc08ad97100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}, 0xc01f41f680, 0xc0ceb17848) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}, 0x3, 0xc05ca447c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc013b97980}}}, {0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}, 0xc01f41f680, 0xc05ca447c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06e8d6090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e8d6090}, {0x40, 0xc013b97b70}, 0xc013b97ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06e8d6090}, {0x4e9d7a0, 0xc08ad97100}, 0xc01f41f680, 0xc0ceb17848) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06e8d6090}, 0xc076dcaae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06526c360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06526c360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219651847 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3dfb90, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0778b1500, 0xc012877800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0778b1500, {0xc012877800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0778b1500, {0xc012877800, 0x739, 0xc04ae412c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc029e1bc28, {0xc012877800, 0x140, 0x13f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc021c22a80, {0xc012877800, 0x2, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0700abe60, {0xc012877800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc050f3ecf8, {0x631b900, 0xc0700abe60}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc050f3ea80, {0x7fe42ab59310, 0xc021c22a80}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc050f3ea80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc050f3ea80, {0xc010109000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0aae2dc20, {0xc010109000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03bfa6088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03bfa6088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03bfa6660, {0x634aac0, 0xc03bfa6088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03bfa6000, 0xc03b5f5880, 0xc07818142b, 0xc07818142d, {0x635bec0, 0xc034b6fef0}, 0xc000d6ae00, 0xc048c5c294, 0xc078181429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03bfa6000, {0x63c85b8, 0xc04e218f00}, 0xc03b5f5870, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08ae35053, 0xb}}, 0x0, 0xc04e218ea0, 0xc04e218ed0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04e218f00}, {0x643d810, 0xc021c22a80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc078402ea0}, {0x643d810, 0xc021c22a80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc078402ea0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808704 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc079bfe270}, 0xc021ff7b60, 0x1, 0xc010cb90a8, 0x519bbc, 0xc038479320, 0xc065130060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc079bfe270}, 0x1e0, 0xc010cb9230, 0x384792c0, 0x0, 0x660069, 0xc038479320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc079bfe270}, 0xc0384792c0, {{{{0x0, 0x0}, {0xc05c483200, 0x125}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc079bfe270}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc079bfe270}, {0xc065cfe000, {0x174301105a2bef15, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc079bfe270}, 0xc0611359a0, 0xc010cb99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc079bfe270}, 0x660089, {0xc065cfe000, {0x174301105a2bef15, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc079bfe270}, 0xc0409d9700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a2bef15, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a2bef15, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079bfe1e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079bfe1e0}, {0xc010cbb3f8, 0x660089}, 0xc010cbb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079bfe1e0}, {0x100a0000100e00}, 0xc0409d9500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079bfe150}, 0xc0409d9500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}, 0xc0327afb20, 0xc053c6ebe8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}, 0x3, 0xc041bcc300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010cbb980}}}, {0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}, 0xc0327afb20, 0xc041bcc300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079bfe150}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079bfe150}, {0x40, 0xc010cbbb70}, 0xc010cbbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079bfe150}, {0x4e9d7a0, 0xc0409d9500}, 0xc0327afb20, 0xc053c6ebe8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079bfe150}, 0xc02b52bce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc083e2be60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc083e2be60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802935 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f22b040, {0x63c85b8, 0xc0cc6e29f0}, 0xc035a19d40, 0x1, 0xc06848d0a8, 0x519bbc, 0xc040027c80, 0xc05056aa80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0cc6e29f0}, 0x120, 0xc06848d230, 0x40027c20, 0x0, 0x660069, 0xc040027c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f22b040, {0x63c85b8, 0xc0cc6e29f0}, 0xc040027c20, {{{{0x0, 0x0}, {0xc073da3e60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0cc6e29f0}, 0x6352410, 0x14c740e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f2199a0, {0x63c85b8, 0xc0cc6e29f0}, {0xc040027b00, {0x174301102488145a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f2199a0, {0x63c85b8, 0xc0cc6e29f0}, 0xc0601d8500, 0xc06848d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0cc6e29f0}, 0x660089, {0xc040027b00, {0x174301102488145a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02072d000, {0x63c85b8, 0xc0cc6e29f0}, 0xc04c523600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102488145a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102488145a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0cc6e2960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0cc6e2960}, {0xc06848f3f8, 0x660089}, 0xc06848f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0cc6e2960}, {0x0}, 0xc04c523400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0cc6e28d0}, 0xc04c523400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}, 0xc06798d140, 0xc018d928d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}, 0x3, 0xc08d7a6f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06848f980}}}, {0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}, 0xc06798d140, 0xc08d7a6f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0cc6e28d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0cc6e28d0}, {0x40, 0xc06848fb70}, 0xc06848fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0cc6e28d0}, {0x4e9d7a0, 0xc04c523400}, 0xc06798d140, 0xc018d928d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0cc6e28d0}, 0xc040ab35c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0400279e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0400279e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808534 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0713b4dd0, {0x63c85b8, 0xc043542720}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc043542720}, {0x4f67ad2, 0x6446ff8}, 0xc016471fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 217838274 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc025fe37d0, 0x107) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0a0512a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0a05130e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc065490000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc065490000, {0x63c8510, 0xc025fe3800}, 0xc061b79dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc021a75f40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc025fe3800}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc021a75f40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811662 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch.func4.1({0x63c8580, 0xc03a352360}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:301 +0xf6 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc084bc4690}, {0x4f0948f, 0x11}, 0x4a817c800, 0xc002d99ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch.func4({0x63c85b8, 0xc084bc4690}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:299 +0x91 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812619 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e034f00, {0x63c85b8, 0xc0304cb980}, 0xc031cfb5d8, 0x1, 0x7080183b7a1393, 0x37e11d600, 0xc07258a780, 0xc042cc5860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc047c6c6c0, {0x63c85b8, 0xc0304cb980}, 0xc047c6c710, 0xc04243f1a8, 0x41, 0x60, 0xffffffffffffffff, 0xc07258a780, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e034f00, {0x63c85b8, 0xc0304cb980}, 0xc07258a6c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc05eb8f8c0, 0x8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0304cb980}, 0x6352260, 0x47c52650) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd310, {0x63c85b8, 0xc0304cb980}, {0xc07258a5a0, {0x174301105aa31fd2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd310, {0x63c85b8, 0xc0304cb980}, 0xc071187360, 0xc04243f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0304cb980}, 0x660089, {0xc07258a5a0, {0x174301105aa31fd2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc0304cb980}, 0xc0460df600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aa31fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aa31fd2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0304cb8c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304cb8c0}, {0xc0424413f8, 0x660089}, 0xc042441400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0304cb8c0}, {0x39362d363765612d}, 0xc0460df400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0304cb6e0}, 0xc0460df400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}, 0xc03ebdd920, 0xc031cfb5a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}, 0x3, 0xc015b9e6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042441980}}}, {0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}, 0xc03ebdd920, 0xc015b9e6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0304cb6e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304cb6e0}, {0x40, 0xc042441b70}, 0xc042441ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0304cb6e0}, {0x4e9d7a0, 0xc0460df400}, 0xc03ebdd920, 0xc031cfb5a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0304cb6e0}, 0xc039f0bf80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 212779529 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc03ac97610, 0xed) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc06893ea80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc06893f0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05705f300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05705f300, {0x63c8510, 0xc03ac97640}, 0xc07c133dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03251ab00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03ac97640}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03251ab00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802462 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc02ecdc7e0}, {0xc061157e40, 0x2, 0xc09c0c7e58}, {0x0, {0x1743010ff2198aa1, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc02ecdc7e0}, 0x18701, 0xc04aed10e0, 0x2b, 0xc0538596e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc02ecdc7e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805116 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc060d17110}, 0xc0d35fec90, 0x1, 0x0, 0x1, 0xc066e0e3c0, 0xc0512a2060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc060d17110}, 0x51cdcd8, 0xc0936bc728, 0x8da96630, 0x4c166a, 0x18, 0xc066e0e3c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc060d17110}, 0xc066e0e360, {{{{0x0, 0x0}, {0xc0477b4ea0, 0x19}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc060d17110}, 0x6352410, 0x1f8e1610) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc060d17110}, {0xc0722aea20, {0x174301101b746c81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc060d17110}, 0xc039c88640, 0xc08da96ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc060d17110}, 0x660089, {0xc0722aea20, {0x174301101b746c81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc060d17110}, 0xc06c4b6300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b746c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b746c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc060d17080}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc060d17080}, {0xc0936928f0, 0x660089}, 0xc0936928f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc060d17080}, {0xc00102e4e0}, 0xc06c4b6200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc060d17020}, 0xc06c4b6200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06c2f91a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809159 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03bd19180, {0x63c85b8, 0xc036d38750}, 0xc04b0e6048, 0x1, 0xc0259830a8, 0x519bbc, 0xc07977bbc0, 0xc07553fce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0186265a0, {0x63c85b8, 0xc036d38750}, 0xc018626600, 0xc0259831a8, 0x45, 0x60, 0xffffffffffffffff, 0xc07977bbc0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03bd19180, {0x63c85b8, 0xc036d38750}, 0xc07977bb00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc084426900, 0xa}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc036d38750}, 0x63521a0, 0x4b1bc1a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc067985680, {0x63c85b8, 0xc036d38750}, {0xc07977b9e0, {0x1743010fdc409ece, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc067985680, {0x63c85b8, 0xc036d38750}, 0xc077a2c5a0, 0xc0259839f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc036d38750}, 0x660089, {0xc07977b9e0, {0x1743010fdc409ece, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc036d38750}, 0xc043b2a200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fdc409ece, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fdc409ece, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc036d38690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc036d38690}, {0xc0259853f8, 0x660089}, 0xc025985400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc036d38690}, {0x0}, 0xc051eb1f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc036d38600}, 0xc051eb1f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}, 0xc0ce3150e0, 0xc01e72e948) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}, 0x3, 0xc03943de00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc025985980}}}, {0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}, 0xc0ce3150e0, 0xc03943de00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc036d38600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc036d38600}, {0x40, 0xc025985b70}, 0xc025985ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc036d38600}, {0x4e9d7a0, 0xc051eb1f00}, 0xc0ce3150e0, 0xc01e72e948) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc036d38600}, 0xc0757bafc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07977b8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07977b8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803855 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc03a0157d0}, 0xc03f82b398, 0x1, 0xc0465970a8, 0x519bbc, 0xc06440c720, 0xc03a3c63c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03a0157d0}, 0x120, 0xc046597230, 0x6440c6c0, 0x0, 0x660069, 0xc06440c720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc03a0157d0}, 0xc06440c6c0, {{{{0x0, 0x0}, {0xc0631c2fc0, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03a0157d0}, 0x6352410, 0x34331260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc03a0157d0}, {0xc06440c5a0, {0x1743011015406c30, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc03a0157d0}, 0xc04d9e4960, 0xc0465979f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03a0157d0}, 0x660089, {0xc06440c5a0, {0x1743011015406c30, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc03a0157d0}, 0xc0a4f85f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011015406c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011015406c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03a0156e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a0156e0}, {0xc0465993f8, 0x660089}, 0xc046599400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03a0156e0}, {0xfa79734d19660001}, 0xc0a4f85d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a015650}, 0xc0a4f85d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}, 0xc039a58280, 0xc028551a58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}, 0x3, 0xc044f49bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc046599980}}}, {0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}, 0xc039a58280, 0xc044f49bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03a015650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a015650}, {0x40, 0xc046599b70}, 0xc046599ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03a015650}, {0x4e9d7a0, 0xc0a4f85d00}, 0xc039a58280, 0xc028551a58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03a015650}, 0xc06be25f20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc064c83200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc064c83200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803425 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc021a2bc20}, 0xc05c1ff800, 0xc081abc960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc021a2bc20}, 0xc05c1ff800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e330700, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e330700, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021a2bb90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021a2bb90}, {0xc076c693f8, 0x660089}, 0xc076c69400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021a2bb90}, {0x0}, 0xc05c1ff500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021a2bb00}, 0xc05c1ff500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}, 0xc03d18a420, 0xc030069a58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}, 0x3, 0xc08689ce40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076c69980}}}, {0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}, 0xc03d18a420, 0xc08689ce40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc021a2bb00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021a2bb00}, {0x40, 0xc076c69b70}, 0xc076c69ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc021a2bb00}, {0x4e9d7a0, 0xc05c1ff500}, 0xc03d18a420, 0xc030069a58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc021a2bb00}, 0xc075bb2600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041e02240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041e02240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804943 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0573ce2d0}, 0xc0299a5f08, 0x1, 0xc0955010a8, 0x519bbc, 0xc058e023c0, 0xc070de2cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0573ce2d0}, 0x4bf403, 0xc095501230, 0x58e02420, 0xc058e02360, 0x1, 0xc058e023c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0573ce2d0}, 0xc058e02360, {{{{0x0, 0x0}, {0xc04a324b40, 0xf5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0573ce2d0}, 0x6352410, 0x3f6a17f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0573ce2d0}, {0xc058e02240, {0x174301103f561c59, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0573ce2d0}, 0xc085a16aa0, 0xc0955019f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0573ce2d0}, 0x660089, {0xc058e02240, {0x174301103f561c59, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0573ce2d0}, 0xc07031d200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f561c59, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f561c59, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0573ce240}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0573ce240}, {0xc0955033f8, 0x660089}, 0xc095503400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0573ce240}, {0x7fe3ba761d08}, 0xc07031cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0573ce1b0}, 0xc07031cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}, 0xc07ba14fe0, 0xc0180aaa80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}, 0x3, 0xc03e525dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc095503980}}}, {0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}, 0xc07ba14fe0, 0xc03e525dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0573ce1b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0573ce1b0}, {0x40, 0xc095503b70}, 0xc095503ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}, 0x20300f, 0x20300f) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0573ce1b0}, {0x4e9d7a0, 0xc07031cf00}, 0xc07ba14fe0, 0xc0180aaa80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0573ce1b0}, 0xc027801200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06f9cf440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06f9cf440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808493 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0533ce780}, 0xc02c7ea9f0, 0x1, 0xc0084cd0a8, 0x519bbc, 0xc05b281260, 0xc05eb18600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0533ce780}, 0x120, 0xc0084cd230, 0x5b281200, 0x0, 0x660069, 0xc05b281260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0533ce780}, 0xc05b281200, {{{{0x0, 0x0}, {0xc051c4afc0, 0x122}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0533ce780}, 0x6352410, 0x32431600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0533ce780}, {0xc05b2810e0, {0x174301105d4b31f8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0533ce780}, 0xc025c36d20, 0xc0084cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0533ce780}, 0x660089, {0xc05b2810e0, {0x174301105d4b31f8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0533ce780}, 0xc0c8bdb200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105d4b31f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105d4b31f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0533ce6f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0533ce6f0}, {0xc0084cf3f8, 0x660089}, 0xc0084cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0533ce6f0}, {0x3}, 0xc0c8bdb000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0533ce660}, 0xc0c8bdb000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}, 0xc0d475be80, 0xc06582b9b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}, 0x3, 0xc058e08900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0084cf980}}}, {0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}, 0xc0d475be80, 0xc058e08900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0533ce660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0533ce660}, {0x40, 0xc0084cfb70}, 0xc0084cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0533ce660}, {0x4e9d7a0, 0xc0c8bdb000}, 0xc0d475be80, 0xc06582b9b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0533ce660}, 0xc02023b4a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b280fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b280fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219114287 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc084ec8490, 0x22c) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc058ccf500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc058ccfb60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0eb288c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0eb288c00, {0x63c8510, 0xc084ec84c0}, 0xc0bbce9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03bf178c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc084ec84c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03bf178c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219805644 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc022e84330}, 0xc02a0b2a38, 0x1, 0xc005b590a8, 0x519bbc, 0xc08bd841e0, 0xc02a255e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0bab75e60, {0x63c85b8, 0xc022e84330}, 0xc0765b2ea0, 0xc005b591a8, 0x8bd841e0, 0x0, 0xffffffffffffffff, 0xc08bd841e0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc022e84330}, 0xc08bd84120, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0831fd680, 0x77}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc022e84330}, 0x6352260, 0x1cdd97e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc022e84330}, {0xc08bd84000, {0x1743010ff7584398, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc022e84330}, 0xc04a8cf900, 0xc005b599f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022e84330}, 0x660089, {0xc08bd84000, {0x1743010ff7584398, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc022e84330}, 0xc063448b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff7584398, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff7584398, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022e841e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022e841e0}, {0xc005b5b3f8, 0x660089}, 0xc005b5b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022e841e0}, {0x0}, 0xc063448800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022e84150}, 0xc063448800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}, 0xc06deb8600, 0xc07d344cd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}, 0x3, 0xc0639a3140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005b5b980}}}, {0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}, 0xc06deb8600, 0xc0639a3140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022e84150}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022e84150}, {0x40, 0xc005b5bb70}, 0xc005b5bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022e84150}, {0x4e9d7a0, 0xc063448800}, 0xc06deb8600, 0xc07d344cd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022e84150}, 0xc0499e1f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04e46be60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04e46be60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806984 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f1b9c00, {0x63c85b8, 0xc03af976e0}, 0xc07d41e700, 0xc0564a0fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f1b9c00, {0x63c85b8, 0xc03af976e0}, 0xc07d41e700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011042c871a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011042c871a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03af97650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03af97650}, {0xc0051613f8, 0x660089}, 0xc005161400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03af97650}, {0x0}, 0xc07d41e400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03af975c0}, 0xc07d41e400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}, 0xc090c21000, 0xc0202306f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}, 0x3, 0xc090051c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005161980}}}, {0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}, 0xc090c21000, 0xc090051c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03af975c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03af975c0}, {0x40, 0xc005161b70}, 0xc005161ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03af975c0}, {0x4e9d7a0, 0xc07d41e400}, 0xc090c21000, 0xc0202306f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03af975c0}, 0xc04c216d20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0718bde60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0718bde60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802473 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc047498e40}, 0xc059201000, 0xc0141dde00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc047498e40}, 0xc059201000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffb08eaae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffb08eaae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047498db0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047498db0}, {0xc054fd73f8, 0x660089}, 0xc054fd7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047498db0}, {0x0}, 0xc059200d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047498cc0}, 0xc059200d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}, 0xc08c9288e0, 0xc01ff39470) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}, 0x3, 0xc0d43eea00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc054fd7980}}}, {0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}, 0xc08c9288e0, 0xc0d43eea00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047498cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047498cc0}, {0x40, 0xc054fd7b70}, 0xc054fd7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047498cc0}, {0x4e9d7a0, 0xc059200d00}, 0xc08c9288e0, 0xc01ff39470) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047498cc0}, 0xc047214d80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04b5c3680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04b5c3680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794223 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b2f40, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc078a5fb00, 0xc0060e2800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc078a5fb00, {0xc0060e2800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc078a5fb00, {0xc0060e2800, 0x735, 0xc04a73b540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc055c36b40, {0xc0060e2800, 0x7eda79, 0xc0060e2805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06e8805a0, {0xc0060e2800, 0xf83fb4, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc049654f30, {0xc0060e2800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc082105078, {0x631b900, 0xc049654f30}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc082104e00, {0x7fe42ab59310, 0xc06e8805a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc082104e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc082104e00, {0xc03a317000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0715c8a20, {0xc03a317000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04400ab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04400ab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04400b0e0, {0x634aac0, 0xc04400ab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04400aa80, 0xc0343c1600, 0xc03f95142b, 0xc03f95142d, {0x635bec0, 0xc04c5270e0}, 0xc000d6ae00, 0xc086f79138, 0xc03f951429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04400aa80, {0x63c85b8, 0xc077824870}, 0xc0343c15f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc062e6b038, 0x8}}, 0x0, 0xc0778247e0, 0xc077824840, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc077824870}, {0x643d810, 0xc06e8805a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc040453440}, {0x643d810, 0xc06e8805a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc040453440}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810351 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00d7f8000, {0x63c85b8, 0xc086df1c50}, 0xc08ba3fb78, 0x1, 0xc00b4c90a8, 0x519bbc, 0xc08677bda0, 0xc047529020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc086df1c50}, 0x120, 0xc00b4c9230, 0x8677bd40, 0x0, 0x660069, 0xc08677bda0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00d7f8000, {0x63c85b8, 0xc086df1c50}, 0xc08677bd40, {{{{0x0, 0x0}, {0xc05a73c480, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc086df1c50}, 0x6352410, 0x50b34e80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc008377770, {0x63c85b8, 0xc086df1c50}, {0xc08677bc20, {0x174301106d6d1d47, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc008377770, {0x63c85b8, 0xc086df1c50}, 0xc0794245a0, 0xc00b4c99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc086df1c50}, 0x660089, {0xc08677bc20, {0x174301106d6d1d47, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000dd5c00, {0x63c85b8, 0xc086df1c50}, 0xc062d46f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d6d1d47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d6d1d47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086df1bc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086df1bc0}, {0xc00b4cb3f8, 0x660089}, 0xc00b4cb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086df1bc0}, {0xc0015221e0}, 0xc062d46d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086df1b30}, 0xc062d46d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}, 0xc020f94040, 0xc03ed25a70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}, 0x3, 0xc039cba000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b4cb980}}}, {0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}, 0xc020f94040, 0xc039cba000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086df1b30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086df1b30}, {0x40, 0xc00b4cbb70}, 0xc00b4cbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086df1b30}, {0x4e9d7a0, 0xc062d46d00}, 0xc020f94040, 0xc03ed25a70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086df1b30}, 0xc0449c70e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0483c8a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0483c8a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802040 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b683a40, {0x63c85b8, 0xc0672c0720}, 0xc023846600, 0x1, 0xc04fb990a8, 0x519bbc, 0xc039578d80, 0xc0282e3aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0672c0720}, 0x1e0, 0xc04fb99230, 0x39578d20, 0x0, 0x660069, 0xc039578d80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b683a40, {0x63c85b8, 0xc0672c0720}, 0xc039578d20, {{{{0x0, 0x0}, {0xc03ea12fc0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0672c0720}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b5e9d60, {0x63c85b8, 0xc0672c0720}, {0xc036ad3c20, {0x1743011018399f42, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b5e9d60, {0x63c85b8, 0xc0672c0720}, 0xc0781e32c0, 0xc04fb999f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0672c0720}, 0x660089, {0xc036ad3c20, {0x1743011018399f42, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b6e0000, {0x63c85b8, 0xc0672c0720}, 0xc059de9e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011018399f42, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011018399f42, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0672c0690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0672c0690}, {0xc04fb9b3f8, 0x660089}, 0xc04fb9b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0672c0690}, {0x7fe3bdd5f5d0}, 0xc059de9c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0672c0600}, 0xc059de9c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}, 0xc01d93fce0, 0xc03db7a4c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}, 0x3, 0xc02068ba80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04fb9b980}}}, {0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}, 0xc01d93fce0, 0xc02068ba80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0672c0600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0672c0600}, {0x40, 0xc04fb9bb70}, 0xc04fb9bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0672c0600}, {0x4e9d7a0, 0xc059de9c00}, 0xc01d93fce0, 0xc03db7a4c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0672c0600}, 0xc01f3a5ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc058ffd7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc058ffd7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803418 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021c38400, {0x63c85b8, 0xc074590e70}, 0xc0d503e300, 0xc04d8185a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c38400, {0x63c85b8, 0xc074590e70}, 0xc0d503e300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d76df70, 0x0, 0x0}, 0xc046af15a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d76df70, 0x0, 0x0}, 0xc046af15a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074590de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074590de0}, {0xc076857918, 0x660089}, 0xc076857920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074590de0}, {0xc00102e4e0}, 0xc0d503e200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074590d80}, 0xc0d503e200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc04c551bc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc074590cf0}, {0xc00dfd36b0, 0x24}, 0x46c7c2540, 0xc0385e7d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc06cf183f0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc074590cf0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807759 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc054707920}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219766162 [runnable]: internal/poll.runtime_pollWait(0x7fde186b2e50, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc02dafdb00, 0xc0511c6000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02dafdb00, {0xc0511c6000, 0x1855, 0x1855}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc02dafdb00, {0xc0511c6000, 0x1855, 0xc0557dcb40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0537e6f18, {0xc0511c6000, 0x7eda79, 0xc0511c6005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04f7e6e40, {0xc0511c6000, 0x2, 0x1855}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0342fdb90, {0xc0511c6000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc09a9ebaf8, {0x631b900, 0xc0342fdb90}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc09a9eb880, {0x7fe42ab59310, 0xc04f7e6e40}, 0x1855) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc09a9eb880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc09a9eb880, {0xc08e419000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc070823b18, {0xc08e419000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06f79c088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06f79c088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06f79c660, {0x634aac0, 0xc06f79c088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06f79c000, 0xc0190b7350, 0xc075fcd42b, 0xc075fcd42d, {0x635bec0, 0xc05ae3e090}, 0xc000d6ae00, 0xc066a30248, 0xc075fcd429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06f79c000, {0x63c85b8, 0xc06cc93e60}, 0xc0190b7340, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc03ab7c037, 0x8}}, 0x0, 0xc06cc93e00, 0xc06cc93e30, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06cc93e60}, {0x643d810, 0xc04f7e6e40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06ac8c7e0}, {0x643d810, 0xc04f7e6e40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06ac8c7e0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219493042 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe7a308, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0c88fc680, 0xc0612fe000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0c88fc680, {0xc0612fe000, 0x2b8e, 0x2b8e}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0c88fc680, {0xc0612fe000, 0x2b8e, 0xc0629a72c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc055181518, {0xc0612fe000, 0x7eda79, 0xc0612fe005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06e633260, {0xc0612fe000, 0x3630623635363612, 0x2b8e}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02902d0f8, {0xc0612fe000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0b8a26cf8, {0x631b900, 0xc02902d0f8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0b8a26a80, {0x7fe42ab59310, 0xc06e633260}, 0x2b8e) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0b8a26a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0b8a26a80, {0xc028b56000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc07554a5d0, {0xc028b56000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc097098b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc097098b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0970990e0, {0x634aac0, 0xc097098b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc097098a80, 0xc03da60340, 0xc06dfd542b, 0xc06dfd542d, {0x635bec0, 0xc0432cd4d0}, 0xc000d6ae00, 0xc040baa148, 0xc06dfd5429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc097098a80, {0x63c85b8, 0xc050329ce0}, 0xc03da60330, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc028b4a037, 0x8}}, 0x0, 0xc050329c80, 0xc050329cb0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc050329ce0}, {0x643d810, 0xc06e633260}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0264ffbf0}, {0x643d810, 0xc06e633260}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0264ffbf0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811904 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff68280, {0x63c85b8, 0xc066630060}, 0xc0cd98db00, 0x1, 0xc0294750a8, 0x519bbc, 0xc05052f5c0, 0xc09c9f4060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc066630060}, 0x120, 0xc029475230, 0x5052f560, 0x0, 0x660069, 0xc05052f5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff68280, {0x63c85b8, 0xc066630060}, 0xc05052f560, {{{{0x0, 0x0}, {0xc03f084c60, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc066630060}, 0x6352410, 0x36ce8080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea3e00, {0x63c85b8, 0xc066630060}, {0xc05052f440, {0x174301107d502aaa, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea3e00, {0x63c85b8, 0xc066630060}, 0xc064891d60, 0xc0294759f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066630060}, 0x660089, {0xc05052f440, {0x174301107d502aaa, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff72000, {0x63c85b8, 0xc066630060}, 0xc04f4b9400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107d502aaa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107d502aaa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06fc6bfb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fc6bfb0}, {0xc0294773f8, 0x660089}, 0xc029477400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06fc6bfb0}, {0x7fe323bd1f50}, 0xc04f4b9200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fc6bef0}, 0xc04f4b9200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}, 0xc0405011a0, 0xc0254ecca8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}, 0x3, 0xc06aff2300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc029477980}}}, {0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}, 0xc0405011a0, 0xc06aff2300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06fc6bef0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fc6bef0}, {0x40, 0xc029477b70}, 0xc029477ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06fc6bef0}, {0x4e9d7a0, 0xc04f4b9200}, 0xc0405011a0, 0xc0254ecca8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06fc6bef0}, 0xc016f1b860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc019e578c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc019e578c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812378 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021c6c800, {0x63c85b8, 0xc07d3266f0}, 0xc0a5abf500, 0xc0852f1cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c6c800, {0x63c85b8, 0xc07d3266f0}, 0xc0a5abf500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07d326660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07d326660}, {0xc085a79858, 0x660089}, 0xc085a79860) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07d326660}, {0xc00102e4e0}, 0xc0a5abf400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07d326600}, 0xc0a5abf400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x174301108372327a, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc00f726a20}, 0x0, 0xc061a76b00) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc00f726a20}, 0x1e) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxnRecover(0xc00d502dc0, {0x63c85b8, 0xc00f726a20}, 0xc0171ce480, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:304 +0x135 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxn.func2({0x63c85b8, 0xc00f726a20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:160 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00f726a20}, {0x10000c012930000, 0x20}, 0xc097200ee0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxn(0xc00d502dc0, 0xc0171ce480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:133 +0x133 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).ResolveIndeterminateCommit.func1() github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:96 +0x25 github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).doCall(0xc00d502e40, 0xc06235a1e0, {0xc073069830, 0x24}, 0xc097200fb8) github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:128 +0x3b created by github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).DoChan github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:121 +0x2f1 goroutine 219802476 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024eb0800, {0x63c85b8, 0xc047e32c00}, 0xc046fdc400, 0xc0474d4c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024eb0800, {0x63c85b8, 0xc047e32c00}, 0xc046fdc400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025560e06, 0x0, 0x0}, 0xc03060afd0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025560e06, 0x0, 0x0}, 0xc03060afd0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e32b70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e32b70}, {0xc063e173f8, 0x660089}, 0xc063e17400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e32b70}, {0xc0015221e0}, 0xc046fdc200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e324b0}, 0xc046fdc200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}, 0xc0968d0620, 0xc06316fb60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}, 0x3, 0xc056d17380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc063e17980}}}, {0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}, 0xc0968d0620, 0xc056d17380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047e324b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e324b0}, {0x40, 0xc063e17b70}, 0xc063e17ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047e324b0}, {0x4e9d7a0, 0xc046fdc200}, 0xc0968d0620, 0xc06316fb60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047e324b0}, 0xc0974e2180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04732bb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04732bb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812734 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc01fce02c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc078810000, 0xc042131168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01fce02c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc078810000, {0xc0ce0f2cb0, 0x28, 0x70}, {0xc01e845ea0, 0x28, 0x4b}, 0x0, 0xd9) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc096d7d7e0, {0xc0ce0f2cb0, 0x1, 0x4b}, {0xc01e845ea0, 0xc096d7ca00, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc096d7d7e0, {0xc0ce0f2cb0, 0x28, 0x70}, {0xc01e845ea0, 0x28, 0x4b}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc096d7ccc0, {0xc01e845ea0, 0x0, 0xc0421314e0}, 0x121ea30, 0x20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc096d7ccc0, {0xc0ce0f2cb0, 0x28, 0x4b}, {0xc01e845ea0, 0x27, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc096d7ca00, {0xc01e845ea0, 0x28, 0x4b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc051d5b608, {{0xc051de1ce0, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc059c67200, {{0xc051de1ce0, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08f3f7000, {0x63c85b8, 0xc062cdf830}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062cdf830}, {0x64a6560, 0xc059c67200}, {0xc051de1ce0, 0x6, 0x30}, {0x174301109a744e9e, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062cdf830}, {0x6466ed8, 0xc051d5b600}, {0xc051de1ce0, 0x27, 0x30}, {0x174301109a744e9e, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04dbe5f68}, {{0x174301109a744e9e, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109a744e9e, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03362e600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b841b39e, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02f355c00, {0x63c85b8, 0xc062cdf830}, 0xc03362e600, 0xc01bce3a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f355c00, {0x63c85b8, 0xc062cdf830}, 0xc03362e600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109a744e9e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109a744e9e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cdf7a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdf7a0}, {0xc0421373f8, 0x660089}, 0xc042137400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cdf7a0}, {0x0}, 0xc03362e200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cdf710}, 0xc03362e200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}, 0xc06ac12e40, 0xc0490965a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}, 0x3, 0xc0b08ec740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042137980}}}, {0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}, 0xc06ac12e40, 0xc0b08ec740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cdf710}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdf710}, {0x40, 0xc042137b70}, 0xc042137ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cdf710}, {0x4e9d7a0, 0xc03362e200}, 0xc06ac12e40, 0xc0490965a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cdf710}, 0xc085102720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f9680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f9680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808420 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff68280, {0x63c85b8, 0xc018161230}, 0xc046a94378, 0x1, 0xc0107550a8, 0x519bbc, 0xc059e99800, 0xc049337b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc018161230}, 0x120, 0xc010755230, 0x59e997a0, 0x0, 0x660069, 0xc059e99800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff68280, {0x63c85b8, 0xc018161230}, 0xc059e997a0, {{{{0x0, 0x0}, {0xc00d3ebd40, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc018161230}, 0x6352230, 0x1b929928) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea3e00, {0x63c85b8, 0xc018161230}, {0xc059e99680, {0x174301105c38c3e5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea3e00, {0x63c85b8, 0xc018161230}, 0xc03ea2cf00, 0xc0107559f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc018161230}, 0x660089, {0xc059e99680, {0x174301105c38c3e5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff72000, {0x63c85b8, 0xc018161230}, 0xc0c9185500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c38c3e5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c38c3e5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0181611a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0181611a0}, {0xc0107573f8, 0x660089}, 0xc010757400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0181611a0}, {0x40e11478e29b1739}, 0xc0c9185300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018160f30}, 0xc0c9185300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}, 0xc03a377060, 0xc0232398c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}, 0x3, 0xc05b18bf80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010757980}}}, {0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}, 0xc03a377060, 0xc05b18bf80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc018160f30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018160f30}, {0x40, 0xc010757b70}, 0xc010757ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc018160f30}, {0x4e9d7a0, 0xc0c9185300}, 0xc03a377060, 0xc0232398c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc018160f30}, 0xc072691680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059e990e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059e990e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810323 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01edcb400, {0x63c85b8, 0xc07a44c180}, 0xc064713600, 0xc02e2f2dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc07a44c180}, 0xc064713600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c65bb38, 0x0, 0x0}, 0xc03fc6eba0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c65bb38, 0x0, 0x0}, 0xc03fc6eba0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a44c0f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a44c0f0}, {0xc025be73f8, 0x660089}, 0xc025be7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a44c0f0}, {0x7fe3f73d2378}, 0xc064713400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a44c060}, 0xc064713400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}, 0xc088711820, 0xc047b81008) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}, 0x3, 0xc0118bd6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc025be7980}}}, {0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}, 0xc088711820, 0xc0118bd6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a44c060}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a44c060}, {0x40, 0xc025be7b70}, 0xc025be7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a44c060}, {0x4e9d7a0, 0xc064713400}, 0xc088711820, 0xc047b81008) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a44c060}, 0xc017017da0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0216a19e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0216a19e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806090 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc04eb338f0}, 0xc054358800, 0xc018f339a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04eb338f0}, 0xc054358800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104b712bcb, 0x0, 0x0}, 0xc0ae6695a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104b712bcb, 0x0, 0x0}, 0xc0ae6695a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04eb33860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04eb33860}, {0xc00642f3f8, 0x660089}, 0xc00642f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04eb33860}, {0xc07b8f21c0}, 0xc054358600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04eb337d0}, 0xc054358600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}, 0xc0309f7660, 0xc084acbdd0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}, 0x3, 0xc00f3de2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00642f980}}}, {0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}, 0xc0309f7660, 0xc00f3de2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04eb337d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04eb337d0}, {0x40, 0xc00642fb70}, 0xc00642fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04eb337d0}, {0x4e9d7a0, 0xc054358600}, 0xc0309f7660, 0xc084acbdd0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04eb337d0}, 0xc01b695440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07762efc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07762efc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805488 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc06c45b1a0}, 0xc0734d6c60, 0x1, 0xc0047390a8, 0x519bbc, 0xc0910f04e0, 0xc05fcb75c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06c45b1a0}, 0x120, 0xc004739230, 0x910f0480, 0x0, 0x660069, 0xc0910f04e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc06c45b1a0}, 0xc0910f0480, {{{{0x0, 0x0}, {0xc05fc898c0, 0x56}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06c45b1a0}, 0x6352410, 0x4040c4c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc06c45b1a0}, {0xc093ad3e60, {0x17430110449da1e7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc06c45b1a0}, 0xc072b908c0, 0xc0047399f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06c45b1a0}, 0x660089, {0xc093ad3e60, {0x17430110449da1e7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc06c45b1a0}, 0xc0653e4300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110449da1e7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110449da1e7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044033da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044033da0}, {0xc00473b3f8, 0x660089}, 0xc00473b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044033da0}, {0x7fe429e694d8}, 0xc036124000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044033d10}, 0xc036124000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}, 0xc073d89fe0, 0xc02844bdb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}, 0x3, 0xc0562dc5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00473b980}}}, {0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}, 0xc073d89fe0, 0xc0562dc5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044033d10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044033d10}, {0x40, 0xc00473bb70}, 0xc00473bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044033d10}, {0x4e9d7a0, 0xc036124000}, 0xc073d89fe0, 0xc02844bdb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044033d10}, 0xc022357ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093ad3d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093ad3d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812600 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810303 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc075d76d20}, 0xc085be0330, 0x1, 0xc003ed90a8, 0x519bbc, 0xc08537b2c0, 0xc056fdef60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc07d47d8c0, {0x63c85b8, 0xc075d76d20}, 0xc07d47d900, 0xc003ed91a8, 0x27, 0x40, 0xffffffffffffffff, 0xc08537b2c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc075d76d20}, 0xc08537b200, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0a98ced80, 0x44}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc075d76d20}, 0x6352260, 0x1baa29f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc075d76d20}, {0xc08537b0e0, {0x174301103208748e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc075d76d20}, 0xc07639e0a0, 0xc003ed99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc075d76d20}, 0x660089, {0xc08537b0e0, {0x174301103208748e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc075d76d20}, 0xc03d7dd600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103208748e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103208748e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075d769f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075d769f0}, {0xc003edb3f8, 0x660089}, 0xc003edb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075d769f0}, {0x7fe39116b330}, 0xc03d7dd400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc075d76960}, 0xc03d7dd400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}, 0xc064569220, 0xc014904720) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}, 0x3, 0xc039c3ae80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003edb980}}}, {0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}, 0xc064569220, 0xc039c3ae80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc075d76960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075d76960}, {0x40, 0xc003edbb70}, 0xc003edbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc075d76960}, {0x4e9d7a0, 0xc03d7dd400}, 0xc064569220, 0xc014904720) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc075d76960}, 0xc02da1b7a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc08537afc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc08537afc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811171 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc046387560}, 0xc03c92b9e0, 0x1, 0xc027e250a8, 0x519bbc, 0xc01cd6d020, 0xc0485762a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046387560}, 0x120, 0xc027e25230, 0x1cd6cfc0, 0x0, 0x660069, 0xc01cd6d020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc046387560}, 0xc01cd6cfc0, {{{{0x0, 0x0}, {0xc061a45680, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046387560}, 0x6352410, 0x1978a880) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc046387560}, {0xc01cd6cd80, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc046387560}, 0xc07c1fb7c0, 0xc027e259f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046387560}, 0x660089, {0xc01cd6cd80, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc046387560}, 0xc072b39b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046387470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046387470}, {0xc027e273f8, 0x660089}, 0xc027e27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046387470}, {0x0}, 0xc072b39900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0463873e0}, 0xc072b39900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}, 0xc01ed827e0, 0xc03a388348) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}, 0x3, 0xc047938f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc027e27980}}}, {0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}, 0xc01ed827e0, 0xc047938f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0463873e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0463873e0}, {0x40, 0xc027e27b70}, 0xc027e27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0463873e0}, {0x4e9d7a0, 0xc072b39900}, 0xc01ed827e0, 0xc03a388348) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0463873e0}, 0xc01cd8d020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc021842c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc021842c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802686 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc026155dd0}, 0xc0b4d7b400, 0xc0688a4000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc026155dd0}, 0xc0b4d7b400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffea6e970, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffea6e970, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc026155d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc026155d40}, {0xc0501b93f8, 0x660089}, 0xc0501b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc026155d40}, {0x0}, 0xc0b4d7b200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc026155c50}, 0xc0b4d7b200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}, 0xc0278e0800, 0xc041c2be78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}, 0x3, 0xc02375f7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0501b9980}}}, {0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}, 0xc0278e0800, 0xc02375f7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc026155c50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc026155c50}, {0x40, 0xc0501b9b70}, 0xc0501b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc026155c50}, {0x4e9d7a0, 0xc0b4d7b200}, 0xc0278e0800, 0xc041c2be78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc026155c50}, 0xc043ecab40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807650 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e284f00, {0x63c85b8, 0xc029551f20}, 0xc08bb52f48, 0x1, 0xc0042570a8, 0x519bbc, 0xc058536a00, 0xc0702e8780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc044135d40, {0x63c85b8, 0xc029551f20}, 0xc058536a00, 0xc0042571a8, 0x64, 0xa0, 0x0, 0xc058536a00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e284f00, {0x63c85b8, 0xc029551f20}, 0xc058536700, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc02118c5a0, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc029551f20}, 0x63521a0, 0x4040d770) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e2a4500, {0x63c85b8, 0xc029551f20}, {0xc0783d0900, {0x1743011000ee868e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e2a4500, {0x63c85b8, 0xc029551f20}, 0xc020c62780, 0xc0042579f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc029551f20}, 0x660089, {0xc0783d0900, {0x1743011000ee868e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e2e3000, {0x63c85b8, 0xc029551f20}, 0xc09949ad00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011000ee868e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011000ee868e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029551e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029551e90}, {0xc0042593f8, 0x660089}, 0xc004259400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029551e90}, {0x0}, 0xc09949aa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029551dd0}, 0xc09949aa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}, 0xc084712620, 0xc037aeb1b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}, 0x3, 0xc07beefb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004259980}}}, {0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}, 0xc084712620, 0xc07beefb00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029551dd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029551dd0}, {0x40, 0xc004259b70}, 0xc004259ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029551dd0}, {0x4e9d7a0, 0xc09949aa00}, 0xc084712620, 0xc037aeb1b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029551dd0}, 0xc033f48540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03e451680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03e451680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804588 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc079174a80}, 0xc02c4edf50, 0x1, 0xc0751d70a8, 0x519bbc, 0xc06475acc0, 0xc0522a0f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc079174a80}, 0x120, 0xc0751d7230, 0x6475ac60, 0x0, 0x660069, 0xc06475acc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc079174a80}, 0xc06475ac60, {{{{0x0, 0x0}, {0xc048501d40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc079174a80}, 0x6352410, 0x3d0ac510) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc079174a80}, {0xc06475ab40, {0x1743011031e4a909, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc079174a80}, 0xc077e9f860, 0xc0751d79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc079174a80}, 0x660089, {0xc06475ab40, {0x1743011031e4a909, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc079174a80}, 0xc0701b8100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0791749f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0791749f0}, {0xc0751d93f8, 0x660089}, 0xc0751d9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0791749f0}, {0x7fde186dc8c0}, 0xc02a22bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079174930}, 0xc02a22bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}, 0xc054287f80, 0xc048fc7d28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}, 0x3, 0xc06d761780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0751d9980}}}, {0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}, 0xc054287f80, 0xc06d761780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079174930}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079174930}, {0x40, 0xc0751d9b70}, 0xc0751d9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079174930}, {0x4e9d7a0, 0xc02a22bf00}, 0xc054287f80, 0xc048fc7d28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079174930}, 0xc0abc70de0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0602790e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0602790e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802588 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0261558f0}, 0xc056cc79c8, 0x1, 0xc036efb0a8, 0x519bbc, 0xc03cb8ed80, 0xc038f38060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0261558f0}, 0x1e0, 0xc036efb230, 0x3cb8ed20, 0x0, 0x660069, 0xc03cb8ed80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0261558f0}, 0xc03cb8ed20, {{{{0x0, 0x0}, {0xc08d00c900, 0xcb}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0261558f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0261558f0}, {0xc05c489d40, {0x17430110209f78f6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0261558f0}, 0xc0456554a0, 0xc036efb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0261558f0}, 0x660089, {0xc05c489d40, {0x17430110209f78f6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0261558f0}, 0xc0b4d7ae00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110209f78f6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110209f78f6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc026155860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc026155860}, {0xc036efd3f8, 0x660089}, 0xc036efd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc026155860}, {0xc0015221e0}, 0xc0b4d7ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0261557d0}, 0xc0b4d7ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}, 0xc0278e0680, 0xc041c2bd28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}, 0x3, 0xc02375f4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc036efd980}}}, {0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}, 0xc0278e0680, 0xc02375f4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0261557d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0261557d0}, {0x40, 0xc036efdb70}, 0xc036efdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0261557d0}, {0x4e9d7a0, 0xc0b4d7ac00}, 0xc0278e0680, 0xc041c2bd28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0261557d0}, 0xc043eca5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc038294b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc038294b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 216483508 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc05ffa5cd0, 0x113) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc026185500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc026185b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06159a000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06159a000, {0x63c8510, 0xc05ffa5d00}, 0xc055743dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc043900c50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc05ffa5d00}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc043900c50) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219805714 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e096000, {0x63c85b8, 0xc079f0b6b0}, 0xc04b558500, 0xc0b6ff8dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e096000, {0x63c85b8, 0xc079f0b6b0}, 0xc04b558500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104555bd36, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104555bd36, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079f0b620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079f0b620}, {0xc095a3f3f8, 0x660089}, 0xc095a3f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079f0b620}, {0xc0015221e0}, 0xc04b558200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079f0b560}, 0xc04b558200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}, 0xc00d68cee0, 0xc04c598cf0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}, 0x3, 0xc004ba80c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc095a3f980}}}, {0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}, 0xc00d68cee0, 0xc004ba80c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079f0b560}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079f0b560}, {0x40, 0xc095a3fb70}, 0xc095a3fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}, 0x203001, 0x203001) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079f0b560}, {0x4e9d7a0, 0xc04b558200}, 0xc00d68cee0, 0xc04c598cf0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079f0b560}, 0xc022356060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060690ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060690ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799941 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x6f) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xe0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc038618cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc06e2d7500, {0xc081a97290, 0x32, 0x8d}, {0xc02d3c9c80, 0x32, 0x79}, 0x1, 0x8c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc038618b88, {0xc081a97290, 0x1, 0x79}, {0xc02d3c9c80, 0x28b, 0xc0895c4790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc038618b88, {0xc081a97290, 0x32, 0x8d}, {0xc02d3c9c80, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0386182c0, {0xc02d3c9c80, 0x79, 0x8d}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0386182c0, {0xc081a97290, 0x79, 0xc02d3c9c80}, {0xc02d3c9c80, 0xc065f4eee0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc038618000, {0xc02d3c9c80, 0x32, 0x79}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06c0379a8, {{0xc065f4eee0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc061330d00, {{0xc036925050, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0619b1000, {0x63c85b8, 0xc03d3c4750}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03d3c4750}, {0x64a6560, 0xc061330d00}, {0xc036925050, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03d3c4750}, {0x6466ed8, 0xc06c037700}, {0xc036925050, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc026662af8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03bfc8100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209bdc00, {0x63c85b8, 0xc03d3c4750}, 0xc03bfc8100, 0xc03166f7c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209bdc00, {0x63c85b8, 0xc03d3c4750}, 0xc03bfc8100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03d3c4630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d3c4630}, {0xc0058548f0, 0x660089}, 0xc0058548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03d3c4630}, {0x24}, 0xc03bfc8000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03d3c45d0}, 0xc03bfc8000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01174b890}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790815 [runnable]: sync.runtime_SemacquireMutex(0xc082ca6690, 0x8f, 0x7fe1b200d680) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc082ca6538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc082ca6560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc03694d680, {0xffffffffffffff15, 0xc082ca66c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc03694d680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc03694d680, {0xc0da8a9d80, 0x32, 0x74}, {0xc020150c00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0bfcb8d98, {0xc0da8a9d80, 0x32, 0x74}, {0xc020150c00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0bfcb76c0, {0xc020150c00, 0x72, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0bfcb76c0, {0xc0da8a9d80, 0x72, 0xc020150c00}, {0xc020150c00, 0xc03d2ab200, 0x31}, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0bfcb7400, {0xc020150c00, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04a2ac828, {{0xc03d2ab200, 0x31, 0x40}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc062b2a900, {{0xc01d488960, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc03fdee000, {0x63c85b8, 0xc05f891440}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc05f891440}, {0x64a6560, 0xc062b2a900}, {0xc01d488960, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc05f891440}, {0x6466ed8, 0xc04a2ac580}, {0xc01d488960, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04459e288}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05f8ab100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f754400, {0x63c85b8, 0xc05f891440}, 0xc05f8ab100, 0xc073bba1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f754400, {0x63c85b8, 0xc05f891440}, 0xc05f8ab100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05f8913b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05f8913b0}, {0xc003eea8f0, 0x660089}, 0xc003eea8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05f8913b0}, {0xc00102e4e0}, 0xc05f8ab000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05f891350}, 0xc05f8ab000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc05f891230}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219511881 [semacquire]: sync.runtime_SemacquireMutex(0xc097f51ba0, 0x3, 0xc097f51ba0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074ab70) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc07de98cc0) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074ab60, 0x11869e5, 0x7fe3e7c9f000, 0x8782, 0x7fe16813b700) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc0b3698ec0, 0x7fe3e7c9f001, 0x7fe3e7c9f020, 0x3092, 0x3097) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0140e8000, {0x703b665, 0x3092}, 0x0, 0xc065e1aff8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc065e1ad80, {0xc0, 0x11}, 0x7fe41d484154) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc065e1ad80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).skipForward(0xc065e1ad80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:918 +0x48 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).Next(0xc065e1ad80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:885 +0xf9 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).Next(0xc065e1ad80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1584 +0x33 github.com/cockroachdb/pebble.(*levelIter).Next(0xc07ab36898) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:559 +0x155 github.com/cockroachdb/pebble.(*mergingIter).nextEntry(0xc07ab351c0, 0xc028843200) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:506 +0x6f github.com/cockroachdb/pebble.(*mergingIter).Next(0xc07ab351c0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:1028 +0x4e github.com/cockroachdb/pebble.(*Iterator).nextUserKey(0xc07ab34f00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:480 +0x184 github.com/cockroachdb/pebble.(*Iterator).NextWithLimit(0xc07ab34f00, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1195 +0x45f github.com/cockroachdb/pebble.(*Iterator).Next(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1178 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).Next(0xc03378a900) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:384 +0x48 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).Next(0xc03378a900) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:601 +0x36a github.com/cockroachdb/cockroach/pkg/storage.ComputeStatsForRange({0x6436600, 0xc03378a900}, {0xc03092e828, 0x16, 0x18}, {0xc03092e840, 0x16, 0x18}, 0x0, {0xc07de99920, ...}) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:3798 +0x1e6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sha512(0x9b97ce0, {0x63c8580, 0xc02b8c36e0}, {0x4944, {0xc03092e828, 0x16, 0x18}, {0xc03092e840, 0x16, 0x18}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_consistency.go:658 +0x888 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).computeChecksumPostApply.func1.1({0x63c8580, 0xc02b8c36e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_consistency.go:785 +0x16a github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03ee51e30}, {0x4f77105, 0x23}, 0x34630b8a000, 0xc0a0717e50) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).computeChecksumPostApply.func1({0x63c85b8, 0xc03ee51e30}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_consistency.go:777 +0x116 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812618 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021cd2000, {0x63c85b8, 0xc0304cb4d0}, 0xc031cfb410, 0x1, 0xc0427cb0a8, 0x519bbc, 0xc07258a540, 0xc078783980) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0642b1560, {0x63c85b8, 0xc0304cb4d0}, 0xc0642b1590, 0xc0427cb1a8, 0x30, 0x40, 0xffffffffffffffff, 0xc07258a540, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021cd2000, {0x63c85b8, 0xc0304cb4d0}, 0xc07258a480, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04b5f18c0, 0xd}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0304cb4d0}, 0x6352260, 0x47c52640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c432c0, {0x63c85b8, 0xc0304cb4d0}, {0xc07258a360, {0x174301105a865aee, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c432c0, {0x63c85b8, 0xc0304cb4d0}, 0xc039b4bcc0, 0xc0427cb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0304cb4d0}, 0x660089, {0xc07258a360, {0x174301105a865aee, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd6000, {0x63c85b8, 0xc0304cb4d0}, 0xc0460df300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a865aee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a865aee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0304cb380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304cb380}, {0xc0427cd3f8, 0x660089}, 0xc0427cd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0304cb380}, {0x5524e49736f6e61}, 0xc0460df100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0304cb1d0}, 0xc0460df100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}, 0xc03ebdd8c0, 0xc031cfb548) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}, 0x3, 0xc015b9e600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0427cd980}}}, {0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}, 0xc03ebdd8c0, 0xc015b9e600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0304cb1d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0304cb1d0}, {0x40, 0xc0427cdb70}, 0xc0427cdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0304cb1d0}, {0x4e9d7a0, 0xc0460df100}, 0xc03ebdd8c0, 0xc031cfb548) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0304cb1d0}, 0xc039f0be60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801419 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021c6b000, {0x63c85b8, 0xc020fb4510}, 0xc04947f000, 0xc07aa257c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c6b000, {0x63c85b8, 0xc020fb4510}, 0xc04947f000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffa0bbe50, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffa0bbe50, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020fb4450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020fb4450}, {0xc03a2693f8, 0x660089}, 0xc03a269400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020fb4450}, {0x7fe4292fb800}, 0xc04947ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020fb4360}, 0xc04947ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}, 0xc076a26ee0, 0xc0388e9bd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}, 0x3, 0xc0648404c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03a269980}}}, {0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}, 0xc076a26ee0, 0xc0648404c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020fb4360}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020fb4360}, {0x40, 0xc03a269b70}, 0xc03a269ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020fb4360}, {0x4e9d7a0, 0xc04947ec00}, 0xc076a26ee0, 0xc0388e9bd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020fb4360}, 0xc020d17500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0373377a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0373377a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806006 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc01c568c90}, 0xc04d55c600, 0xc046885360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01c568c90}, 0xc04d55c600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104a9902c9, 0x0, 0x0}, 0xc0744f6380, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104a9902c9, 0x0, 0x0}, 0xc0744f6380, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c568c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c568c00}, {0xc005ae73f8, 0x660089}, 0xc005ae7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c568c00}, {0xc0015221e0}, 0xc04d55c400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c568b70}, 0xc04d55c400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}, 0xc0929cb960, 0xc07995fc80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}, 0x3, 0xc066f92e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ae7980}}}, {0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}, 0xc0929cb960, 0xc066f92e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01c568b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c568b70}, {0x40, 0xc005ae7b70}, 0xc005ae7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01c568b70}, {0x4e9d7a0, 0xc04d55c400}, 0xc0929cb960, 0xc07995fc80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01c568b70}, 0xc044c4b500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04ebe4a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04ebe4a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806970 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc02f4c2990}, 0xc039aa2120, 0x1, 0xc004db50a8, 0x519bbc, 0xc0623da600, 0xc09b52c600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02f4c2990}, 0x120, 0xc004db5230, 0x623da5a0, 0x0, 0x660069, 0xc0623da600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc02f4c2990}, 0xc0623da5a0, {{{{0x0, 0x0}, {0xc0370c5b00, 0xa}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02f4c2990}, 0x6352410, 0x25ab6910) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc02f4c2990}, {0xc0623da480, {0x17430110542a73b6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc02f4c2990}, 0xc02a658460, 0xc004db59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02f4c2990}, 0x660089, {0xc0623da480, {0x17430110542a73b6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc02f4c2990}, 0xc07de3db00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110542a73b6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110542a73b6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f4c28d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f4c28d0}, {0xc004db73f8, 0x660089}, 0xc004db7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f4c28d0}, {0xc0015221e0}, 0xc07de3d900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f4c2840}, 0xc07de3d900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}, 0xc05665f060, 0xc039aa36c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}, 0x3, 0xc0379ec680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004db7980}}}, {0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}, 0xc05665f060, 0xc0379ec680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f4c2840}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f4c2840}, {0x40, 0xc004db7b70}, 0xc004db7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f4c2840}, {0x4e9d7a0, 0xc07de3d900}, 0xc05665f060, 0xc039aa36c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f4c2840}, 0xc08e02d140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0623da360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0623da360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218865235 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4da48, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc072184a00, 0xc00f7fe800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc072184a00, {0xc00f7fe800, 0x746, 0x746}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc072184a00, {0xc00f7fe800, 0x746, 0xc0541b9a40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc025ab7eb8, {0xc00f7fe800, 0x7eda79, 0xc00f7fe805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06b3a26c0, {0xc00f7fe800, 0x629625, 0x746}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02902d128, {0xc00f7fe800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04f277778, {0x631b900, 0xc02902d128}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04f277500, {0x7fe42ab59310, 0xc06b3a26c0}, 0x746) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04f277500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04f277500, {0xc0f09b3000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc053dba3a8, {0xc0f09b3000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0d8486088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0d8486088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0d8486660, {0x634aac0, 0xc0d8486088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0d8486000, 0xc06651c880, 0xc0d1bd342b, 0xc0d1bd342d, {0x635bec0, 0xc06e9fad80}, 0xc000d6ae00, 0xc01fb60634, 0xc0d1bd3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0d8486000, {0x63c85b8, 0xc082be1530}, 0xc06651c870, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0f098903a, 0x18}}, 0x0, 0xc082be14d0, 0xc082be1500, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc082be1530}, {0x643d810, 0xc06b3a26c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc024afaba0}, {0x643d810, 0xc06b3a26c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc024afaba0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219087353 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af29708, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05eb2af00, 0xc05ec91000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05eb2af00, {0xc05ec91000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05eb2af00, {0xc05ec91000, 0x735, 0xc02dd21b80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0418808c8, {0xc05ec91000, 0x7eda79, 0xc05ec91005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01b694b40, {0xc05ec91000, 0x18ba91b, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc047918690, {0xc05ec91000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc068417e78, {0x631b900, 0xc047918690}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc068417c00, {0x7fe42ab59310, 0xc01b694b40}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc068417c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc068417c00, {0xc00daf7000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc07a1f4bd0, {0xc00daf7000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc09176f588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc09176f588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc09176fb60, {0x634aac0, 0xc09176f588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc09176f500, 0xc01e4dfba0, 0xc0936a742b, 0xc0936a742d, {0x635bec0, 0xc06effb5f0}, 0xc000d6ae00, 0xc02db57938, 0xc0936a7429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc09176f500, {0x63c85b8, 0xc0310d2150}, 0xc01e4dfb90, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc011e32009, 0x8}}, 0x0, 0xc0310d20f0, 0xc0310d2120, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0310d2150}, {0x643d810, 0xc01b694b40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0720e8d80}, {0x643d810, 0xc01b694b40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0720e8d80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810224 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01d898800, {0x63c85b8, 0xc062111320}, 0xc02c4f1b00, 0xc03930d900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc062111320}, 0xc02c4f1b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c3ba159, 0x0, 0x0}, 0xc05abdcd10, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c3ba159, 0x0, 0x0}, 0xc05abdcd10, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062111290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062111290}, {0xc01f2093f8, 0x660089}, 0xc01f209400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062111290}, {0x7fe3910102a0}, 0xc02c4f1900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062111200}, 0xc02c4f1900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}, 0xc04ba62600, 0xc048c9bf38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}, 0x3, 0xc070da00c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f209980}}}, {0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}, 0xc04ba62600, 0xc070da00c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062111200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062111200}, {0x40, 0xc01f209b70}, 0xc01f209ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062111200}, {0x4e9d7a0, 0xc02c4f1900}, 0xc04ba62600, 0xc048c9bf38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062111200}, 0xc038afb2c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc02d785320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc02d785320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219790603 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa0a58, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04460f400, 0xc098fe0800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04460f400, {0xc098fe0800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04460f400, {0xc098fe0800, 0x739, 0xc0478b3680}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0450cbd38, {0xc098fe0800, 0x7eda79, 0xc098fe0805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04d653e00, {0xc098fe0800, 0x2, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0c05c3cc8, {0xc098fe0800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0529adaf8, {0x631b900, 0xc0c05c3cc8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0529ad880, {0x7fe42ab59310, 0xc04d653e00}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0529ad880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0529ad880, {0xc00e29e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02a74e5d0, {0xc00e29e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc08fad9588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc08fad9588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc08fad9b60, {0x634aac0, 0xc08fad9588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc08fad9500, 0xc087fe6d30, 0xc0989c942b, 0xc0989c942d, {0x635bec0, 0xc0548a2750}, 0xc000d6ae00, 0xc04ee14778, 0xc0989c9429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc08fad9500, {0x63c85b8, 0xc06d27fdd0}, 0xc087fe6d20, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06cafa059, 0xb}}, 0x0, 0xc06d27fd70, 0xc06d27fda0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06d27fdd0}, {0x643d810, 0xc04d653e00}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03ea006f0}, {0x643d810, 0xc04d653e00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03ea006f0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807675 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e04d400, {0x63c85b8, 0xc0861a1aa0}, 0xc08922a200, 0xc06e2b25a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc0861a1aa0}, 0xc08922a200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101e73bb1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101e73bb1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0861a19b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0861a19b0}, {0xc00ad673f8, 0x660089}, 0xc00ad67400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0861a19b0}, {0x7fe4293f15c0}, 0xc08922a000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0861a18f0}, 0xc08922a000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}, 0xc026922b20, 0xc02855fd58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}, 0x3, 0xc01ac9a600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ad67980}}}, {0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}, 0xc026922b20, 0xc01ac9a600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0861a18f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0861a18f0}, {0x40, 0xc00ad67b70}, 0xc00ad67ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0861a18f0}, {0x4e9d7a0, 0xc08922a000}, 0xc026922b20, 0xc02855fd58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0861a18f0}, 0xc0451bc900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc078b9e120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc078b9e120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805985 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021cd2000, {0x63c85b8, 0xc067bb3ad0}, 0xc03c3a3b60, 0x1, 0xc0054b30a8, 0x519bbc, 0xc0650d2240, 0xc06475b080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc02c25ad80, {0x63c85b8, 0xc067bb3ad0}, 0xc0650d2240, 0xc0054b31a8, 0x30, 0x40, 0x3, 0xc0650d2240, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021cd2000, {0x63c85b8, 0xc067bb3ad0}, 0xc0650d2000, {{{{0xc046ed67e0, 0x2}, {0xc08537b9e0, 0x2}}, {{0xc0d6109320, 0x8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc067bb3ad0}, 0x63521a0, 0x4207e510) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c432c0, {0x63c85b8, 0xc067bb3ad0}, {0xc06bd42900, {0x1743010ffa9efcb6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c432c0, {0x63c85b8, 0xc067bb3ad0}, 0xc04fdbbd60, 0xc0054b39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067bb3ad0}, 0x660089, {0xc06bd42900, {0x1743010ffa9efcb6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd6000, {0x63c85b8, 0xc067bb3ad0}, 0xc054de9e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffa9efcb6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffa9efcb6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067bb3a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067bb3a40}, {0xc0054b53f8, 0x660089}, 0xc0054b5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067bb3a40}, {0xc08676b700}, 0xc054de9c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067bb3980}, 0xc054de9c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}, 0xc0309f6220, 0xc0778e7920) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}, 0x3, 0xc066748a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0054b5980}}}, {0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}, 0xc0309f6220, 0xc066748a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc067bb3980}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067bb3980}, {0x40, 0xc0054b5b70}, 0xc0054b5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc067bb3980}, {0x4e9d7a0, 0xc054de9c00}, 0xc0309f6220, 0xc0778e7920) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc067bb3980}, 0xc07a8ddf20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c2b4480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c2b4480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809636 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021c68c00, {0x63c85b8, 0xc020bc9740}, 0xc052738700, 0xc045db1400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c68c00, {0x63c85b8, 0xc020bc9740}, 0xc052738700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063b6cbdc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063b6cbdc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020bc9620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020bc9620}, {0xc00b45f3f8, 0x660089}, 0xc00b45f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020bc9620}, {0x7fe3921d9fd8}, 0xc052738400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020bc9410}, 0xc052738400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}, 0xc0472a3340, 0xc0337f92a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}, 0x3, 0xc01b65dd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b45f980}}}, {0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}, 0xc0472a3340, 0xc01b65dd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020bc9410}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020bc9410}, {0x40, 0xc00b45fb70}, 0xc00b45fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020bc9410}, {0x4e9d7a0, 0xc052738400}, 0xc0472a3340, 0xc0337f92a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020bc9410}, 0xc043aceea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc088161680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc088161680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809459 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01f18cc00, {0x63c85b8, 0xc0784ee600}, 0xc023a52100, 0xc06c7e0dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f18cc00, {0x63c85b8, 0xc0784ee600}, 0xc023a52100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106376d365, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106376d365, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0784ee570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0784ee570}, {0xc0268b13f8, 0x660089}, 0xc0268b1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0784ee570}, {0x7fe3f1e3ec20}, 0xc04c743e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0784ee4e0}, 0xc04c743e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}, 0xc05b3b9ae0, 0xc02ca7bd70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}, 0x3, 0xc0abaaeb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0268b1980}}}, {0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}, 0xc05b3b9ae0, 0xc0abaaeb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0784ee4e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0784ee4e0}, {0x40, 0xc0268b1b70}, 0xc0268b1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0784ee4e0}, {0x4e9d7a0, 0xc04c743e00}, 0xc05b3b9ae0, 0xc02ca7bd70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0784ee4e0}, 0xc03f6058c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0b11a2900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0b11a2900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812530 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc045fa7e30}, 0xc0ba4af9f8, 0x1, 0xc03e9970a8, 0x519bbc, 0xc01689a720, 0xc0137dbb60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc045fa7e30}, 0x120, 0xc03e997230, 0x1689a6c0, 0x0, 0x660069, 0xc01689a720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc045fa7e30}, 0xc01689a6c0, {{{{0x0, 0x0}, {0xc0ae26c000, 0x122}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc045fa7e30}, 0x6352410, 0x204d31b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc045fa7e30}, {0xc00cf3be60, {0x174301108384480b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc045fa7e30}, 0xc078e86f00, 0xc03e9979f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc045fa7e30}, 0x660089, {0xc00cf3be60, {0x174301108384480b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc045fa7e30}, 0xc06bbd4f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108384480b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108384480b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045fa7da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045fa7da0}, {0xc03e9993f8, 0x660089}, 0xc03e999400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045fa7da0}, {0x7fe3f33bfba0}, 0xc06bbd4d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045fa7d10}, 0xc06bbd4d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}, 0xc0ccf7f2c0, 0xc038160f60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}, 0x3, 0xc049737c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03e999980}}}, {0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}, 0xc0ccf7f2c0, 0xc049737c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045fa7d10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045fa7d10}, {0x40, 0xc03e999b70}, 0xc03e999ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045fa7d10}, {0x4e9d7a0, 0xc06bbd4d00}, 0xc0ccf7f2c0, 0xc038160f60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045fa7d10}, 0xc021af0f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdcd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdcd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803318 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0bb541da0}, 0xc05c7d6800, 0xc083df4aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0bb541da0}, 0xc05c7d6800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b7eff25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b7eff25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0bb541d10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0bb541d10}, {0xc0472fd3f8, 0x660089}, 0xc0472fd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0bb541d10}, {0x7fe4256582b0}, 0xc05c7d6500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0bb541c80}, 0xc05c7d6500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}, 0xc02982eb20, 0xc02ab21488) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}, 0x3, 0xc025eab180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0472fd980}}}, {0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}, 0xc02982eb20, 0xc025eab180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0bb541c80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0bb541c80}, {0x40, 0xc0472fdb70}, 0xc0472fdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0bb541c80}, {0x4e9d7a0, 0xc05c7d6500}, 0xc02982eb20, 0xc02ab21488) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0bb541c80}, 0xc0347ee600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc035ced200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc035ced200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804653 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc047e9a390}, 0xc02bd9f650, 0x1, 0xc046fe6aa8, 0x0, 0xc06ab79c80, 0xc0772835c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc047e9a390}, 0x120, 0xc07b9af230, 0x6ab79c20, 0xc075a67d40, 0x18, 0xc06ab79c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc047e9a390}, 0xc06ab79c20, {{{{0x0, 0x0}, {0xc052450fc0, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047e9a390}, 0x6352410, 0x421b1610) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc047e9a390}, {0xc06ab79b00, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc047e9a390}, 0xc06869cdc0, 0xc07b9af9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047e9a390}, 0x660089, {0xc06ab79b00, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc047e9a390}, 0xc08dbb2900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e9a300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e9a300}, {0xc07b9b13f8, 0x660089}, 0xc07b9b1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e9a300}, {0xc03b714550}, 0xc08dbb2600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e9a270}, 0xc08dbb2600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}, 0xc013bea180, 0xc0299e2438) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}, 0x3, 0xc03e23e980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b9b1980}}}, {0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}, 0xc013bea180, 0xc03e23e980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047e9a270}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e9a270}, {0x40, 0xc07b9b1b70}, 0xc07b9b1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}, 0x20300f, 0x20300f) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047e9a270}, {0x4e9d7a0, 0xc08dbb2600}, 0xc013bea180, 0xc0299e2438) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047e9a270}, 0xc01c47e3c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06ab799e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06ab799e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812806 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806745 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc012025000, {0x63c85b8, 0xc08c413ef0}, 0xc06b646300, 0xc0381ed860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc012025000, {0x63c85b8, 0xc08c413ef0}, 0xc06b646300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050ac72ba, 0x0, 0x0}, 0xc08a638390, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050ac72ba, 0x0, 0x0}, 0xc08a638390, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08c413e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08c413e60}, {0xc0abc4e8f0, 0x660089}, 0xc0abc4e8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08c413e60}, {0xc00102e4e0}, 0xc06b646100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08c413e00}, 0xc06b646100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc08e3fa1e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807824 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a1b3cc0, {0x63c85b8, 0xc07a415a40}, 0xc04fe46828, 0x1, 0xc0031e90a8, 0x519bbc, 0xc0496cb2a0, 0xc01b3ef320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05ac14d80, {0x63c85b8, 0xc07a415a40}, 0xc05ac14dd0, 0xc0031e91a8, 0x56, 0x60, 0xffffffffffffffff, 0xc0496cb2a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a1b3cc0, {0x63c85b8, 0xc07a415a40}, 0xc0496cb180, {{{{0x0, 0x0}, {0xc0711b70e0, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x60, {0x63c85b8, 0xc07a415a40}, 0x0, 0xf) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a292230, {0x63c85b8, 0xc07a415a40}, {0xc04a718fc0, {0x1743011059e617e9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a292230, {0x63c85b8, 0xc07a415a40}, 0xc02432ee60, 0xc0031e99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a415a40}, 0x660089, {0xc04a718fc0, {0x1743011059e617e9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a288800, {0x63c85b8, 0xc07a415a40}, 0xc04c63d900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011059e617e9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011059e617e9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a4159b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a4159b0}, {0xc0031eb3f8, 0x660089}, 0xc0031eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a4159b0}, {0x7fe42b769ff0}, 0xc0283ede00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a415920}, 0xc0283ede00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}, 0xc06be60580, 0xc030d2d620) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}, 0x3, 0xc064bacac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0031eb980}}}, {0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}, 0xc06be60580, 0xc064bacac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a415920}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a415920}, {0x40, 0xc0031ebb70}, 0xc0031ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a415920}, {0x4e9d7a0, 0xc0283ede00}, 0xc06be60580, 0xc030d2d620) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a415920}, 0xc0651ddec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04a718d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04a718d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805729 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0b00f7830}, 0xc030342870, 0x1, 0xc0071690a8, 0x519bbc, 0xc01bcdad80, 0xc090ea78c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0b00f7830}, 0x1e0, 0xc007169230, 0x1bcdad20, 0x0, 0x660069, 0xc01bcdad80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0b00f7830}, 0xc01bcdad20, {{{{0x0, 0x0}, {0xc09cb78900, 0x5a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0b00f7830}, 0x63521a0, 0x17e2ed60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0b00f7830}, {0xc078989c20, {0x174301104bee1d80, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0b00f7830}, 0xc0b0282b40, 0xc0071699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0b00f7830}, 0x660089, {0xc078989c20, {0x174301104bee1d80, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0b00f7830}, 0xc0a6df8d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104bee1d80, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104bee1d80, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0b00f77a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b00f77a0}, {0xc00716b3f8, 0x660089}, 0xc00716b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0b00f77a0}, {0x0}, 0xc0a6df8800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0b00f7710}, 0xc0a6df8800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}, 0xc0582dbd00, 0xc08a454d68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}, 0x3, 0xc068b49000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00716b980}}}, {0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}, 0xc0582dbd00, 0xc068b49000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0b00f7710}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b00f7710}, {0x40, 0xc00716bb70}, 0xc00716bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0b00f7710}, {0x4e9d7a0, 0xc0a6df8800}, 0xc0582dbd00, 0xc08a454d68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0b00f7710}, 0xc039453260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc078989b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc078989b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801432 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc09791e548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc09791e5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0911ab1f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc051972e00, {0xc012f75300, 0x2b, 0x72}, {0xc036380d90, 0x2b, 0x6d}, 0x1, 0xb1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0911ab088, {0xc012f75300, 0x1, 0x6d}, {0xc036380d90, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0911ab088, {0xc012f75300, 0x2b, 0x72}, {0xc036380d90, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0911aa7c0, {0xc036380d90, 0x6d, 0x72}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0911aa7c0, {0xc012f75300, 0x2b, 0xc036380d90}, {0xc036380d90, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0911aa500, {0xc036380d90, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04a893088, {{0xc0b8c29cb0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0775a2700, {{0xc0b8c29cb0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08258b400, {0x63c85b8, 0xc0784025a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0784025a0}, {0x64a6560, 0xc0775a2700}, {0xc0b8c29cb0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0784025a0}, {0x6466ed8, 0xc04a893080}, {0xc0b8c29cb0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04b1cccd8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc056a2cc00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c93000, {0x63c85b8, 0xc0784025a0}, 0xc056a2cc00, 0xc07722e500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c93000, {0x63c85b8, 0xc0784025a0}, 0xc056a2cc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078402510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078402510}, {0xc0abc4a8f0, 0x660089}, 0xc0abc4a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078402510}, {0xc00102e4e0}, 0xc056a2cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0784024b0}, 0xc056a2cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02ef02420}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808341 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ebc7a40, {0x63c85b8, 0xc07afeaa50}, 0xc0168e9d40, 0x1, 0x0, 0x1, 0xc052152060, 0xc07c299c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07afeaa50}, 0x120, 0xc00feb0728, 0x52152000, 0x0, 0x660069, 0xc052152060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ebc7a40, {0x63c85b8, 0xc07afeaa50}, 0xc052152000, {{{{0x0, 0x0}, {0xc07a1cdb00, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07afeaa50}, 0x6352410, 0x53d3d070) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f582780, {0x63c85b8, 0xc07afeaa50}, {0xc06b98aa20, {0x17430110567df5e4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f582780, {0x63c85b8, 0xc07afeaa50}, 0xc065964be0, 0xc07cdf4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07afeaa50}, 0x660089, {0xc06b98aa20, {0x17430110567df5e4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f5ac400, {0x63c85b8, 0xc07afeaa50}, 0xc078f0ba00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110567df5e4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110567df5e4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07afea9c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07afea9c0}, {0xc0ad83c8f0, 0x660089}, 0xc0ad83c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07afea9c0}, {0xc00102e4e0}, 0xc078f0b900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07afea960}, 0xc078f0b900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01ff75020}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808358 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc079e43b90}, 0xc075336d38, 0x1, 0xc00d9690a8, 0x519bbc, 0xc01ad84f60, 0xc01a01cf60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc079e43b90}, 0x1e0, 0xc00d969230, 0x1ad84f00, 0x0, 0x660069, 0xc01ad84f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc079e43b90}, 0xc01ad84f00, {{{{0x0, 0x0}, {0xc040ffcfc0, 0x4e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc079e43b90}, 0x63521a0, 0x3b957b58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc079e43b90}, {0xc05ac32b40, {0x174301105c286a81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc079e43b90}, 0xc061185b80, 0xc00d9699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc079e43b90}, 0x660089, {0xc05ac32b40, {0x174301105c286a81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc079e43b90}, 0xc06dccc500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c286a81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c286a81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079e43b00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079e43b00}, {0xc00d96b3f8, 0x660089}, 0xc00d96b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079e43b00}, {0x545446444f533955}, 0xc06dccc300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079e43a70}, 0xc06dccc300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}, 0xc06f053ec0, 0xc076ed1b48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}, 0x3, 0xc05ffa4d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00d96b980}}}, {0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}, 0xc06f053ec0, 0xc05ffa4d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079e43a70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079e43a70}, {0x40, 0xc00d96bb70}, 0xc00d96bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079e43a70}, {0x4e9d7a0, 0xc06dccc300}, 0xc06f053ec0, 0xc076ed1b48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079e43a70}, 0xc04154ad20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ac32900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ac32900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807015 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f1c63c0, {0x63c85b8, 0xc0298c0630}, 0xc0234b9b48, 0x1, 0x708017f1fddc45, 0x37e11d600, 0xc034eb9b60, 0xc04374c4e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0298c0630}, 0x120, 0xc005185230, 0x34eb9b00, 0x0, 0x660069, 0xc034eb9b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f1c63c0, {0x63c85b8, 0xc0298c0630}, 0xc034eb9b00, {{{{0x0, 0x0}, {0xc07a90c480, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0298c0630}, 0x6352410, 0x63fdd9d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f663ae0, {0x63c85b8, 0xc0298c0630}, {0xc034eb99e0, {0x1743011042c871a8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f663ae0, {0x63c85b8, 0xc0298c0630}, 0xc0686ee5a0, 0xc0051859f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0298c0630}, 0x660089, {0xc034eb99e0, {0x1743011042c871a8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f1b9c00, {0x63c85b8, 0xc0298c0630}, 0xc07ce06900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011042c871a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011042c871a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0298c0540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0298c0540}, {0xc0051873f8, 0x660089}, 0xc005187400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0298c0540}, {0xc0a1fef6b8e8aa08}, 0xc07ce06700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0298c0450}, 0xc07ce06700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}, 0xc00230e360, 0xc0234b9ae8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}, 0x3, 0xc07623a3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005187980}}}, {0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}, 0xc00230e360, 0xc07623a3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0298c0450}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0298c0450}, {0x40, 0xc005187b70}, 0xc005187ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0298c0450}, {0x4e9d7a0, 0xc07ce06700}, 0xc00230e360, 0xc0234b9ae8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0298c0450}, 0xc05a5a8f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc034eb98c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc034eb98c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219789275 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc048f4d620}, 0xc00678fcb0, {0x646cc48, 0xc048f4d680}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc048f4d620}, 0xedb7a834d, 0xc03ecd1320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc048f4d620}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04d0a5830}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc00678fef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc04d0a5830}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812554 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc00074a800, 0xc096f22700, 0xc041d19168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc096f22700, {0xc054b34780, 0x17, 0xa0}, {0xc0ed38e8c0, 0x17, 0x6f}, 0x0, 0x36) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc043ff3588, {0xc054b34780, 0x1, 0x6f}, {0xc0ed38e8c0, 0xc043ff2a00, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc043ff3588, {0xc054b34780, 0x17, 0xa0}, {0xc0ed38e8c0, 0x17, 0x6f}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc043ff2cc0, {0xc0ed38e8c0, 0x0, 0xc041d194e0}, 0x121ea30, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc043ff2cc0, {0xc054b34780, 0x17, 0x6f}, {0xc0ed38e8c0, 0x16, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc043ff2a00, {0xc0ed38e8c0, 0x17, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04f9d9088, {{0xc0578ee558, 0x16, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc081d59f00, {{0xc0578ee558, 0x16, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc090683800, {0x63c85b8, 0xc056614ae0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056614ae0}, {0x64a6560, 0xc081d59f00}, {0xc0578ee558, 0x0, 0x18}, {0x1743011092b88deb, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056614ae0}, {0x6466ed8, 0xc04f9d9080}, {0xc0578ee558, 0x16, 0x18}, {0x1743011092b88deb, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04612c240}, {{0x1743011092b88deb, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011092b88deb, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc083617a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b085f2eb, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0182b1400, {0x63c85b8, 0xc056614ae0}, 0xc083617a00, 0xc0530a52c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0182b1400, {0x63c85b8, 0xc056614ae0}, 0xc083617a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011092b88deb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011092b88deb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056614a50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614a50}, {0xc041d1f3f8, 0x660089}, 0xc041d1f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056614a50}, {0x3}, 0xc083617800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0566149c0}, 0xc083617800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}, 0xc0ccf7fce0, 0xc038161d58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}, 0x3, 0xc071e54d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc041d1f980}}}, {0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}, 0xc0ccf7fce0, 0xc071e54d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0566149c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0566149c0}, {0x40, 0xc041d1fb70}, 0xc041d1fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0566149c0}, {0x4e9d7a0, 0xc083617800}, 0xc0ccf7fce0, 0xc038161d58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0566149c0}, 0xc071982c60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810137 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02fe28de0}, 0xc0385e2cb0, {0x646cc48, 0xc02fe28e40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02fe28de0}, 0xedb7a8350, 0xc03cf3d680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02fe28de0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc076482a20}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0385e2ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc076482a20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219150675 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff361b0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc003531b80, 0xc0849d2000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc003531b80, {0xc0849d2000, 0x80a, 0x80a}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc003531b80, {0xc0849d2000, 0x57a, 0xc072e2d900}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc025ab7c48, {0xc0849d2000, 0x7eda79, 0xc0849d2295}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc07d03a5a0, {0xc0849d2000, 0x4bf403, 0x80a}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04a6d9a58, {0xc0849d2000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00dd465f8, {0x631b900, 0xc04a6d9a58}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00dd46380, {0x7fe42ab59310, 0xc07d03a5a0}, 0x57a) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00dd46380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00dd46380, {0xc0ab488000, 0x8000, 0x10000a75b000001}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f bufio.(*Reader).Read(0xc05038d5c0, {0xc05c932c80, 0x9, 0xc04a6d9a40}) GOROOT/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x631b600, 0xc05038d5c0}, {0xc05c932c80, 0x9, 0x9}, 0x9) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc05c932c80, 0x9, 0x0}, {0x631b600, 0xc05038d5c0}) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc05c932c40) golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0b00d4d00, 0x10bed2f, 0x63c85b8) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:611 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc000f02e00, {0x6446ff8, 0xc0b00d4d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:906 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() google.golang.org/grpc/external/org_golang_google_grpc/server.go:848 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn google.golang.org/grpc/external/org_golang_google_grpc/server.go:847 +0x185 goroutine 219806755 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0122d5800, {0x63c85b8, 0xc08d5a2360}, 0xc06b646900, 0xc06b1ee140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0122d5800, {0x63c85b8, 0xc08d5a2360}, 0xc06b646900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050b80e16, 0x0, 0x0}, 0xc08a638440, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050b80e16, 0x0, 0x0}, 0xc08a638440, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08d5a22d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d5a22d0}, {0xc0abc4e8f0, 0x660089}, 0xc0abc4e8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08d5a22d0}, {0xc00102e4e0}, 0xc06b646800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08d5a2270}, 0xc06b646800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc08e3fa5a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810802 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01df39800, {0x63c85b8, 0xc044d99e00}, 0xc0b8d00f00, 0xc0aefa9e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01df39800, {0x63c85b8, 0xc044d99e00}, 0xc0b8d00f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032cd4a27, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032cd4a27, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044d99d70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044d99d70}, {0xc019c653f8, 0x660089}, 0xc019c65400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044d99d70}, {0x0}, 0xc0b8d00d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044d99ce0}, 0xc0b8d00d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}, 0xc04ba63000, 0xc0450f4eb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}, 0x3, 0xc070da1a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc019c65980}}}, {0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}, 0xc04ba63000, 0xc070da1a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044d99ce0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044d99ce0}, {0x40, 0xc019c65b70}, 0xc019c65ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044d99ce0}, {0x4e9d7a0, 0xc0b8d00d00}, 0xc04ba63000, 0xc0450f4eb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044d99ce0}, 0xc04424fe00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc02032e900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc02032e900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795609 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fd82400, {0x63c85b8, 0xc046e1a9c0}, 0xc08a8d3e00, 0xc0b16d74a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fd82400, {0x63c85b8, 0xc046e1a9c0}, 0xc08a8d3e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011080a716be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011080a716be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046e1a930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046e1a930}, {0xc086af47f8, 0x660089}, 0xc086af4800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046e1a930}, {0xc00102e4e0}, 0xc08a8d3d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046e1a8d0}, 0xc08a8d3d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc043f57080, {0x63c85b8, 0xc046e1a630}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0c9030600, {0x63c85b8, 0xc046e1a630}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0c9030600, {0x63c8510, 0xc06e107640}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc0c9030600, {0x63c8510, 0xc06e107640}, {0x63f7db8, 0x9bf6f48}, 0xc086af9088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0c9030600, {0x63c85b8, 0xc0419dccc0}, {{0x63f7db8, 0x9bf6f48}, {0xc043b6cd22, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0c9030600, {0x63c85b8, 0xc0419dccc0}, {{0x63f7db8, 0x9bf6f48}, {0xc043b6cd22, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc043b6cd22, 0x6}, 0x0, 0x0}, {0x29f6d450, 0xedb7a8350, 0x0}, {0x29f6d9a4, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0c9030600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0c9030600, {0x63c8510, 0xc085f21040}, 0xc0766b5dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc022b811a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc085f21040}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc022b811a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812445 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc052b26720}, 0xc048fe54b8, 0x1, 0xc03cb3b6d8, 0x519bbc, 0xc0680e92c0, 0xc050ff09c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05aa8f500, {0x63c85b8, 0xc052b26720}, 0xc03ce91320, 0xc03cb3b7d8, 0x680e92c0, 0x0, 0x0, 0xc0680e92c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc052b26720}, 0xc0680e9200, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc03f3f1440, 0x4d}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc052b26720}, 0x6352260, 0x3abf6ab0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc052b26720}, {0xc0680e90e0, {0x174301104c82f06f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc052b26720}, 0xc03dc89220, 0xc03cb3c020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc052b26720}, 0x660089, {0xc0680e90e0, {0x174301104c82f06f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc052b26720}, 0xc0411c6d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c82f06f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c82f06f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc052b26690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052b26690}, {0xc07c607a28, 0x660089}, 0xc07c607a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc052b26690}, {0xc00102e4e0}, 0xc0411c6c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc052b265d0}, 0xc0411c6c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0925b82d0, {0x63c85b8, 0xc052b26540}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0925b82d0, {0x63c85b8, 0xc052b26540}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc052b26540}, {0x4f67ad2, 0x6446ff8}, 0xc0918d4fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219809853 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc063973320}, 0xc028f97518, 0x1, 0xc01c3710a8, 0x519bbc, 0xc07933eba0, 0xc03f6cb140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc063973320}, 0x1e0, 0xc01c371230, 0x7933eb40, 0x0, 0x660069, 0xc07933eba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc063973320}, 0xc07933eb40, {{{{0x0, 0x0}, {0xc084427680, 0x13}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc063973320}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc063973320}, {0xc049d54900, {0x174301100848e686, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc063973320}, 0xc061135540, 0xc01c3719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc063973320}, 0x660089, {0xc049d54900, {0x174301100848e686, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc063973320}, 0xc074122800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100848e686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100848e686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc063973260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063973260}, {0xc01c3733f8, 0x660089}, 0xc01c373400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc063973260}, {0x0}, 0xc074122600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0639731d0}, 0xc074122600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}, 0xc078f0e740, 0xc072e1ef60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}, 0x3, 0xc0293cf640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01c373980}}}, {0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}, 0xc078f0e740, 0xc0293cf640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0639731d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0639731d0}, {0x40, 0xc01c373b70}, 0xc01c373ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0639731d0}, {0x4e9d7a0, 0xc074122600}, 0xc078f0e740, 0xc072e1ef60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0639731d0}, 0xc043a264e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc049d547e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc049d547e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812359 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc05f6accd0, {0x63c85b8, 0xc07d21ede0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc07d21ede0}, {0x4f67ad2, 0x10beec0}, 0xc002c9efb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219810767 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc032db89f0}, 0xc0d04a0690, 0x1, 0x70801814334539, 0x37e11d600, 0xc00b6673e0, 0xc0937329c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc032db89f0}, 0xc0, 0xc03cdd80a8, 0xb667380, 0x0, 0x660069, 0xc00b6673e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc032db89f0}, 0xc00b667380, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc032db89f0}, 0x63524a0, 0x309928d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc032db89f0}, {0x0, {0x17430110779b928c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc032db89f0}, 0xc04eeab680, 0x1) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc032db89f0}, 0x660089, {0x0, {0x17430110779b928c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc032db89f0}, 0xc06bfca300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc032db8960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc032db8960}, {0xc01f21a270, 0x660089}, 0xc01f21a278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc032db8960}, {0xc032db86c0}, 0xc06bfca200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc032db8900}, 0xc06bfca200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x17430110779b928c, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc032db8870}, 0x0, 0xc067daf080) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc032db8870}, 0x66040f) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc019919860, {0x63c85b8, 0xc032db8870}, {{0xaf, 0x47, 0xb9, 0x42, 0xb2, 0x8d, 0x49, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:922 +0x1b0 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc019919860, {0x63c85b8, 0xc032db8870}, 0xc028769e00, 0xc01e7372e0, 0xc04eee8190) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:610 +0xd4a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc032db8870}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc032db87e0}, {{0xc0c89b0d00, 0x24, 0x24}}, 0xc01f21d380) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc019919860, {0x63c85b8, 0xc032db87e0}, 0xc028769e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0301af310, {0x63c85b8, 0xc032db87e0}, {0x0, {0x174301107794730a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc032db87e0}, 0xc067d92dc0, 0xc01f21d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc032db87e0}, 0x660089, {0x0, {0x174301107794730a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc032db87e0}, 0xc06bfca100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107794730a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107794730a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301107794730a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc032db8750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc032db8750}, {0xc01f21f3f8, 0x660089}, 0xc01f21f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc032db8750}, {0x7fe3b59836f0}, 0xc042c8df00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc032db86c0}, 0xc042c8df00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}, 0xc01e7372a0, 0xc0d04a05d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}, 0x3, 0xc0c89b0c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f21f980}}}, {0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}, 0xc01e7372a0, 0xc0c89b0c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc032db86c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc032db86c0}, {0x40, 0xc01f21fb70}, 0xc01f21fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc032db86c0}, {0x4e9d7a0, 0xc042c8df00}, 0xc01e7372a0, 0xc0d04a05d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc032db86c0}, 0xc02cdcacc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc067000480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc067000480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812602 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812282 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01fab9400, {0x63c85b8, 0xc027211710}, 0xc07d477f00, 0xc0669f97c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01fab9400, {0x63c85b8, 0xc027211710}, 0xc07d477f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107dee08fa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107dee08fa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027211680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027211680}, {0xc02f1eb3f8, 0x660089}, 0xc02f1eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027211680}, {0x174300f4a071d600}, 0xc07d477c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0272115f0}, 0xc07d477c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}, 0xc0672dc400, 0xc0442e0c30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}, 0x3, 0xc040bc46c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02f1eb980}}}, {0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}, 0xc0672dc400, 0xc040bc46c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0272115f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0272115f0}, {0x40, 0xc02f1ebb70}, 0xc02f1ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0272115f0}, {0x4e9d7a0, 0xc07d477c00}, 0xc0672dc400, 0xc0442e0c30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0272115f0}, 0xc02f600300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc063b510e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc063b510e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803243 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc01eff9f50}, 0xc04a4a9488, 0x1, 0xc07560d0a8, 0x519bbc, 0xc0453a3c80, 0xc04330faa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc01eff9f50}, 0x1e0, 0xc07560d230, 0x453a3c20, 0x0, 0x660069, 0xc0453a3c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc01eff9f50}, 0xc0453a3c20, {{{{0x0, 0x0}, {0xc062703200, 0x10}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc05197c480, {0x63c85b8, 0xc01eff9f50}, 0xc007c69d70, 0x7560d428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc01eff9f50}, {0xc0368426c0, {0x174301102bd75419, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc01eff9f50}, 0xc068976280, 0xc07560d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01eff9f50}, 0x660089, {0xc0368426c0, {0x174301102bd75419, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc01eff9f50}, 0xc05a615800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bd75419, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bd75419, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01eff9e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01eff9e90}, {0xc07560f3f8, 0x660089}, 0xc07560f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01eff9e90}, {0x0}, 0xc05a615500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01eff9dd0}, 0xc05a615500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}, 0xc08bd6b6e0, 0xc0542d7518) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}, 0x3, 0xc023e29700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07560f980}}}, {0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}, 0xc08bd6b6e0, 0xc023e29700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01eff9dd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01eff9dd0}, {0x40, 0xc07560fb70}, 0xc07560fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01eff9dd0}, {0x4e9d7a0, 0xc05a615500}, 0xc08bd6b6e0, 0xc0542d7518) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01eff9dd0}, 0xc0385ab3e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0368425a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0368425a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812278 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016f5da40, {0x63c85b8, 0xc03f781620}, 0xc04c38e420, 0x1, 0xc036cdb0a8, 0x519bbc, 0xc09d7762a0, 0xc02edc24e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03f781620}, 0x120, 0xc036cdb230, 0x9d776240, 0x0, 0x660069, 0xc09d7762a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016f5da40, {0x63c85b8, 0xc03f781620}, 0xc09d776240, {{{{0x0, 0x0}, {0xc0785ca120, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03f781620}, 0x6352410, 0x65f8a558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016fda500, {0x63c85b8, 0xc03f781620}, {0xc09d776120, {0x174301107e50a6b9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016fda500, {0x63c85b8, 0xc03f781620}, 0xc03b365180, 0xc036cdb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03f781620}, 0x660089, {0xc09d776120, {0x174301107e50a6b9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017050000, {0x63c85b8, 0xc03f781620}, 0xc04983cc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03f781560}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f781560}, {0xc036cdd3f8, 0x660089}, 0xc036cdd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03f781560}, {0x7fe392087d68}, 0xc04983ca00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03f7814d0}, 0xc04983ca00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}, 0xc03294f5e0, 0xc04c38e4c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}, 0x3, 0xc037015d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc036cdd980}}}, {0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}, 0xc03294f5e0, 0xc037015d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03f7814d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f7814d0}, {0x40, 0xc036cddb70}, 0xc036cddab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03f7814d0}, {0x4e9d7a0, 0xc04983ca00}, 0xc03294f5e0, 0xc04c38e4c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03f7814d0}, 0xc035311200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09fd4d8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09fd4d8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794467 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0476ca660}, 0xc0185c9cb0, {0x646cc48, 0xc0476ca720}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0476ca660}, 0xedb7a834e, 0xc05b981200) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0476ca660}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0393db0e0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0185c9ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0393db0e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804908 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc030af4dc0, {0x63c85b8, 0xc039645b60}, 0xc03a23c138, 0x1, 0xc05530d0a8, 0x519bbc, 0xc058cb7c80, 0xc0634b4960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc039645b60}, 0x120, 0xc05530d230, 0x58cb7c20, 0xc095f5e820, 0x18, 0xc058cb7c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc030af4dc0, {0x63c85b8, 0xc039645b60}, 0xc058cb7c20, {{{{0x0, 0x0}, {0xc04fbe85a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc039645b60}, 0x6352410, 0x32c55170) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0308a02d0, {0x63c85b8, 0xc039645b60}, {0xc058cb7b00, {0x1743011035513431, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0308a02d0, {0x63c85b8, 0xc039645b60}, 0xc06289afa0, 0xc05530d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc039645b60}, 0x660089, {0xc058cb7b00, {0x1743011035513431, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030b12000, {0x63c85b8, 0xc039645b60}, 0xc08a2aee00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011035513431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011035513431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039645ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039645ad0}, {0xc05530f3f8, 0x660089}, 0xc05530f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039645ad0}, {0xc0015221e0}, 0xc08a2aec00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039645a40}, 0xc08a2aec00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}, 0xc03aff06a0, 0xc034bd7788) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}, 0x3, 0xc067615840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05530f980}}}, {0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}, 0xc03aff06a0, 0xc067615840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc039645a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039645a40}, {0x40, 0xc05530fb70}, 0xc05530fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}, 0x203019, 0x203019) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc039645a40}, {0x4e9d7a0, 0xc08a2aec00}, 0xc03aff06a0, 0xc034bd7788) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc039645a40}, 0xc036630420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058c1fc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058c1fc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804812 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc011fee000, {0x63c85b8, 0xc073b6db00}, 0xc053390558, 0x1, 0xc075f45ff0, 0x519bbc, 0xc06d3b6720, 0xc04e0fb4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc073b6db00}, 0x120, 0xc075f46178, 0x6d3b66c0, 0xc095f5eb60, 0x18, 0xc06d3b6720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc011fee000, {0x63c85b8, 0xc073b6db00}, 0xc06d3b66c0, {{{{0x0, 0x0}, {0xc03d292120, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073b6db00}, 0x6352410, 0x199f0570) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc007eeb9f0, {0x63c85b8, 0xc073b6db00}, {0xc06d3b65a0, {0x17430110310ddab1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc007eeb9f0, {0x63c85b8, 0xc073b6db00}, 0xc054ef6f00, 0xc064c36938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073b6db00}, 0x660089, {0xc06d3b65a0, {0x17430110310ddab1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000c55800, {0x63c85b8, 0xc073b6db00}, 0xc041739c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110310ddab1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073b6da70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073b6da70}, {0xc064c38340, 0x660089}, 0xc064c38348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073b6da70}, {0xc00102e4e0}, 0xc041739b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073b6da10}, 0xc041739b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc03a743740}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810264 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc02dfbf500}, 0xc020fb7f20, 0x1, 0xc01f2070a8, 0x519bbc, 0xc06fba0ba0, 0xc03efed380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02dfbf500}, 0x120, 0xc01f207230, 0x6fba0b40, 0x0, 0x660069, 0xc06fba0ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc02dfbf500}, 0xc06fba0b40, {{{{0x0, 0x0}, {0xc03393a6c0, 0xf9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02dfbf500}, 0x6352410, 0x43a28fb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc02dfbf500}, {0xc06fba0a20, {0x174301106bf5c87f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc02dfbf500}, 0xc07531d2c0, 0xc01f2079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02dfbf500}, 0x660089, {0xc06fba0a20, {0x174301106bf5c87f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc02dfbf500}, 0xc059955000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bf5c87f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bf5c87f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02dfbf410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02dfbf410}, {0xc01f2093f8, 0x660089}, 0xc01f209400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02dfbf410}, {0xc0015221e0}, 0xc059954e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02dfbf230}, 0xc059954e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}, 0xc00f3f42e0, 0xc03135a738) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}, 0x3, 0xc063d6ad80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01f209980}}}, {0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}, 0xc00f3f42e0, 0xc063d6ad80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02dfbf230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02dfbf230}, {0x40, 0xc01f209b70}, 0xc01f209ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02dfbf230}, {0x4e9d7a0, 0xc059954e00}, 0xc00f3f42e0, 0xc03135a738) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02dfbf230}, 0xc0bb53e2a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06fba0900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06fba0900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812036 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc025360c80, {0x63c85b8, 0xc03298d680}, 0xc063fae2d0, 0x1, 0x708018193d87ea, 0x37e11d600, 0xc02895c600, 0xc048253500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03298d680}, 0x1e0, 0xc036088728, 0x2895c5a0, 0x0, 0x660069, 0xc02895c600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc025360c80, {0x63c85b8, 0xc03298d680}, 0xc02895c5a0, {{{{0x0, 0x0}, {0xc05261e900, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03298d680}, 0x6352410, 0x33a922c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528fdb0, {0x63c85b8, 0xc03298d680}, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528fdb0, {0x63c85b8, 0xc03298d680}, 0xc06029a960, 0xc058160ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03298d680}, 0x660089, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02540fc00, {0x63c85b8, 0xc03298d680}, 0xc04c24b800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03298d5f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03298d5f0}, {0xc003eee8f0, 0x660089}, 0xc003eee8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03298d5f0}, {0xc00102e4e0}, 0xc04c24b600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03298d590}, 0xc04c24b600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01f1b7740}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 214515099 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc06f853440) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0cd4ff4a0, {0x4d83cc0, 0xc06481af80}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xf5) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc06f8530e0, 0xc084413138, 0xc084413108) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc06f8530e0, {0x4d83cc0, 0xc06481af80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc042b87530}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc050f94800}, {0x4d83cc0, 0xc06481af80}, 0x2, {0xc012856840, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc042b87530}, {0x4f6406d, 0x7fe457d7ff18}, {0x4e9d7a0, 0xc050f94800}, {0x4d83cc0, 0xc06481af80}, 0x7fb9733b8d9efca9, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x30e737582e92bf13, {0x63c85b8, 0xc042b87530}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc050f94800}, {0x4d83cc0, 0xc06481af80}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc033a92f20, {0x63c85b8, 0xc042b87530}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc033a92f20}}, {0x63c85b8, 0xc042b87530}, 0xc050f94800, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1c, 0x1c, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc08a8fc870, {0x63c85b8, 0xc064b5ec30}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc08a8fc870, {0x63c85b8, 0xc064b5ec30}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc018a35130, {0x63c85b8, 0xc064b5ec30}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc06d346000, {0x63c85b8, 0xc064b5ec30}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc06bb1ac60) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:264 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc052ad8800) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc064b5eba0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/is_null_ops.eg.go:226 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc097092f00) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc0501308f0, {0x63c8510, 0xc052ad8780}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0a4a0f270, {0x63c8510, 0xc052ad8780}, 0xc04090a160) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc052ad8500}, 0xc08ca2e380, 0xc04090a160, 0xc076596700, 0xc04abb4000, 0xc04479dce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc052ad8500}, 0xc04479d9f0, 0xc08ca2e380, 0xc0236d9bc0, {{0x63ca340, 0xc076596680}, 0x0}, 0xc04abb4000) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc04479d300, {0x63c8510, 0xc052ad8500}, 0xc04479d9f0, 0xc052ad8500, {0x7fe42b790cf0, 0xc0236d9bc0}, 0x0, 0xc01e015d38) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc04479d300, {0x63c8510, 0xc052ad8500}, 0xc04479d9f0, {0x7fe42b790cf0, 0xc0236d9bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc04479d300, {0x63c85b8, 0xc06be67e60}, {{0x63f8d78, 0xc058f18e60}, {0xc041f5e00f, 0x6a}, 0x1, 0x1}, 0xc06d8ee6c0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc04479d300, {0x63c85b8, 0xc06be67e60}, {{0x63f8d78, 0xc058f18e60}, {0xc041f5e00f, 0x6a}, 0x1, 0x1}, 0xc06d8ee6c0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc04479d300, {0x63c85b8, 0xc06be67e60}, {0xc06d8ee6c0, {0xc0292b97a0, 0x1, 0x1}, {0xc03e209620, 0xd, 0xd}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc08454ee83, 0x0}, 0x0, {0xbe3a82c, 0xedb7a8351, 0x0}, 0x1}, 0xc04479d300, 0xc084419980, 0xc084419970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04479d300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04479d300, {0x63c8510, 0xc00eea90c0}, 0xc05072bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04207d180) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc00eea90c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04207d180) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812702 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806862 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc025d797c0, {0x63c85b8, 0xc09133e4b0}, 0xc049e1fcf8, 0x1, 0x708017ef339cba, 0x37e11d600, 0xc0a2e36820, 0xc034d0d320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0639ce240, {0x63c85b8, 0xc09133e4b0}, 0xc0639ce290, 0xc00355b1a8, 0x56, 0x60, 0xffffffffffffffff, 0xc0a2e36820, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc025d797c0, {0x63c85b8, 0xc09133e4b0}, 0xc0a2e36700, {{{{0xc051d39560, 0x4}, {0xc076750d80, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x60, {0x63c85b8, 0xc09133e4b0}, 0x0, 0xb) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc025d3d090, {0x63c85b8, 0xc09133e4b0}, {0xc0633d5320, {0x17430110514af0b8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc025d3d090, {0x63c85b8, 0xc09133e4b0}, 0xc086c6df40, 0xc00355b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc09133e4b0}, 0x660089, {0xc0633d5320, {0x17430110514af0b8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025dbd400, {0x63c85b8, 0xc09133e4b0}, 0xc0975eb500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110514af0b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110514af0b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09133e2d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09133e2d0}, {0xc00355d3f8, 0x660089}, 0xc00355d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09133e2d0}, {0x0}, 0xc0975eb000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09133e240}, 0xc0975eb000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}, 0xc08eb79e40, 0xc049e1fcc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}, 0x3, 0xc08f0c8e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00355d980}}}, {0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}, 0xc08eb79e40, 0xc08f0c8e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09133e240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09133e240}, {0x40, 0xc00355db70}, 0xc00355dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09133e240}, {0x4e9d7a0, 0xc0975eb000}, 0xc08eb79e40, 0xc049e1fcc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09133e240}, 0xc05ef320c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04fb567e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04fb567e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219214597 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc020906310, 0x37) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc05ff3c000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc05ff3c660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08db94000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08db94000, {0x63c8510, 0xc020906380}, 0xc0b18dfdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07946d330) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc020906380}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07946d330) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812301 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0302d2a00, {0x63c85b8, 0xc02b53c5a0}, 0xc024451860, 0x1, 0x7080181c02faf5, 0x37e11d600, 0xc06d4384e0, 0xc04c8f0f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02b53c5a0}, 0x120, 0xc038c1b230, 0x6d438480, 0x0, 0x660069, 0xc06d4384e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0302d2a00, {0x63c85b8, 0xc02b53c5a0}, 0xc06d438480, {{{{0x0, 0x0}, {0xc05f6d1d40, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02b53c5a0}, 0x6352410, 0x6b066d30) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030161e00, {0x63c85b8, 0xc02b53c5a0}, {0xc06d438360, {0x174301107f2561ee, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030161e00, {0x63c85b8, 0xc02b53c5a0}, 0xc022383540, 0xc038c1b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02b53c5a0}, 0x660089, {0xc06d438360, {0x174301107f2561ee, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0302e5000, {0x63c85b8, 0xc02b53c5a0}, 0xc05f880f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f2561ee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f2561ee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b53c480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b53c480}, {0xc038c1d3f8, 0x660089}, 0xc038c1d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b53c480}, {0x3deb7cdfd9d7bdbb}, 0xc05f880d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02b53c1e0}, 0xc05f880d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}, 0xc078fbd860, 0xc024451830) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}, 0x3, 0xc05c937140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc038c1d980}}}, {0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}, 0xc078fbd860, 0xc05c937140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02b53c1e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b53c1e0}, {0x40, 0xc038c1db70}, 0xc038c1dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02b53c1e0}, {0x4e9d7a0, 0xc05f880d00}, 0xc078fbd860, 0xc024451830) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02b53c1e0}, 0xc05734e840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06d438240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06d438240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809491 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0403ab080}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807455 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0207c4dc0, {0x63c85b8, 0xc038cd7080}, 0xc0437b7ce0, 0x1, 0xc0068376d8, 0x519bbc, 0xc046250f60, 0xc034e771a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06b114000, {0x63c85b8, 0xc038cd7080}, 0xc046250f60, 0xc0068377d8, 0x44, 0x60, 0x0, 0xc046250f60, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0207c4dc0, {0x63c85b8, 0xc038cd7080}, 0xc046250ea0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04432c7e0, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc038cd7080}, 0x6352260, 0x32430748) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0207ce550, {0x63c85b8, 0xc038cd7080}, {0xc046250c60, {0x1743011019292be5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0207ce550, {0x63c85b8, 0xc038cd7080}, 0xc057f90b40, 0xc006838020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc038cd7080}, 0x660089, {0xc046250c60, {0x1743011019292be5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cac00, {0x63c85b8, 0xc038cd7080}, 0xc0986ff900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc038cd6f90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc038cd6f90}, {0xc07c607a28, 0x660089}, 0xc07c607a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc038cd6f90}, {0xc00102e4e0}, 0xc0986ff800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc038cd6f30}, 0xc0986ff800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0555838d0, {0x63c85b8, 0xc038cd6ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0555838d0, {0x63c85b8, 0xc038cd6ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc038cd6ea0}, {0x4f67ad2, 0x6446ff8}, 0xc0062bffb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219800137 [semacquire]: sync.runtime_SemacquireMutex(0xc00074ab70, 0x20, 0x11840a5) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).Lock(0xc0955a2358) GOROOT/src/sync/rwmutex.go:116 +0x71 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074ab60, 0x11869e5, 0x7fe17cf59000, 0x7dda, 0x7fdda490c980) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc078d0d140, 0x7fe17cf59001, 0x7fe17cf59020, 0x2694, 0x2699) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0586cedc0, {0x19c0c94, 0x2694}, 0x0, 0xc0846e4978) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0846e4700, {0xffffffffffffff15, 0xc0955a25e0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0846e4700) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0846e4700, {0xc0031469a0, 0x2b, 0x6d}, 0xc022147280, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0846e4700, {0xc022147280, 0x2b, 0x71}, {0xc0031469a0, 0x2b, 0x6d}, 0x1, 0x8a) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc043ff88e8, {0xc022147280, 0x1, 0x6d}, {0xc0031469a0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc043ff88e8, {0xc022147280, 0x2b, 0x71}, {0xc0031469a0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc043ff76c0, {0xc0031469a0, 0x6d, 0x71}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc043ff76c0, {0xc022147280, 0x2b, 0xc0031469a0}, {0xc0031469a0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc043ff7400, {0xc0031469a0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc054120b08, {{0xc08c5015c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0472abb00, {{0xc08c5015c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc048a0b400, {0x63c85b8, 0xc03d40da40}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03d40da40}, {0x64a6560, 0xc0472abb00}, {0xc08c5015c0, 0x21036, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03d40da40}, {0x6466ed8, 0xc054120b00}, {0xc08c5015c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06edf1218}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc017fcf900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a32400, {0x63c85b8, 0xc03d40da40}, 0xc017fcf900, 0xc0507d3a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a32400, {0x63c85b8, 0xc03d40da40}, 0xc017fcf900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03d40d9b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d40d9b0}, {0xc0076ca8f0, 0x660089}, 0xc0076ca8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03d40d9b0}, {0xc00102e4e0}, 0xc017fcf800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03d40d950}, 0xc017fcf800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d3b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 214020671 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4d958, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06da41f00, 0xc064e62800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06da41f00, {0xc064e62800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06da41f00, {0xc064e62800, 0x5, 0xc06d108bc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0506337d8, {0xc064e62800, 0x140, 0x13f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc07b2491a0, {0xc064e62800, 0x4bf403, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc049a036b0, {0xc064e62800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04c53ccf8, {0x631b900, 0xc049a036b0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04c53ca80, {0x7fe42ab59310, 0xc07b2491a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04c53ca80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04c53ca80, {0xc0a68ba000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02201c7b0, {0xc0a68ba000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a377ab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a377ab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a377b0e0, {0x634aac0, 0xc0a377ab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a377aa80, 0xc02fbb3780, 0xc06d10942b, 0xc06d10942d, {0x635bec0, 0xc0776e0bd0}, 0xc000d6ae00, 0xc03172d928, 0xc06d109429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a377aa80, {0x63c85b8, 0xc044ff9470}, 0xc02fbb3770, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc072e74009, 0x8}}, 0x0, 0xc044ff9410, 0xc044ff9440, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc044ff9470}, {0x643d810, 0xc07b2491a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07543bf80}, {0x643d810, 0xc07b2491a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07543bf80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810401 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01df39800, {0x63c85b8, 0xc086e8fd40}, 0xc062d47800, 0xc0484210e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01df39800, {0x63c85b8, 0xc086e8fd40}, 0xc062d47800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032cb16fd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032cb16fd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086e8fcb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086e8fcb0}, {0xc01654d3f8, 0x660089}, 0xc01654d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086e8fcb0}, {0x7fe3b7aba060}, 0xc062d47500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086e8fc20}, 0xc062d47500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}, 0xc020f946a0, 0xc076f9e858) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}, 0x3, 0xc039cbab80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01654d980}}}, {0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}, 0xc020f946a0, 0xc039cbab80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086e8fc20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086e8fc20}, {0x40, 0xc01654db70}, 0xc01654dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086e8fc20}, {0x4e9d7a0, 0xc062d47500}, 0xc020f946a0, 0xc076f9e858) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086e8fc20}, 0xc0d07822a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0e57c7c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0e57c7c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791855 [runnable]: sync.runtime_SemacquireMutex(0x7fe18dcfdae0, 0x78, 0x118663c) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074aa90, 0x7, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc02a99e800, 0x32, 0x792f9f2edc07, 0x1) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc059ab38c0, {0x81, 0xc081c22740}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc01c102d80, {0xc02a99e800, 0x2b, 0x74}, {0xc057190990, 0x2b, 0x81}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1340 +0xed github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc08e51b740, {0xc02a99e800, 0x2b, 0x74}, {0xc057190990, 0x2b, 0x81}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc08e51a2c0, {0xc057190990, 0x81, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc08e51a2c0, {0xc02a99e800, 0x2b, 0xc057190990}, {0xc057190990, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc08e51a000, {0xc057190990, 0x2b, 0x81}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc059834008, {{0xc0707ae240, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc056438500, {{0xc0707ae240, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc049360c00, {0x63c85b8, 0xc074d665a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc074d665a0}, {0x64a6560, 0xc056438500}, {0xc0707ae240, 0x7171717171717171, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc074d665a0}, {0x6466ed8, 0xc059834000}, {0xc0707ae240, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0672e96e0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06d618a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f9d0000, {0x63c85b8, 0xc074d665a0}, 0xc06d618a00, 0xc017408d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f9d0000, {0x63c85b8, 0xc074d665a0}, 0xc06d618a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074d66510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074d66510}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074d66510}, {0xc00102e4e0}, 0xc06d618900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074d664b0}, 0xc06d618900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc053ec6b10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809800 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0201e3b80, {0x63c85b8, 0xc07bb0a7b0}, 0xc04f2bf1e8, 0x1, 0xc013baf6d8, 0x519bbc, 0xc0829c9740, 0xc039ecb500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0784f4ea0, {0x63c85b8, 0xc07bb0a7b0}, 0xc0829c9740, 0xc013baf7d8, 0x44, 0x60, 0x0, 0xc0829c9740, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0201e3b80, {0x63c85b8, 0xc07bb0a7b0}, 0xc0829c9680, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc057369d40, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07bb0a7b0}, 0x6352260, 0x6b245338) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01f8c5950, {0x63c85b8, 0xc07bb0a7b0}, {0xc0829c9560, {0x174301102cffdd45, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01f8c5950, {0x63c85b8, 0xc07bb0a7b0}, 0xc070e260a0, 0xc013bb0020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07bb0a7b0}, 0x660089, {0xc0829c9560, {0x174301102cffdd45, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020435400, {0x63c85b8, 0xc07bb0a7b0}, 0xc0917c7900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cffdd45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cffdd45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07bb0a720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bb0a720}, {0xc078183a28, 0x660089}, 0xc078183a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07bb0a720}, {0xc00102e4e0}, 0xc0917c7800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07bb0a6c0}, 0xc0917c7800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc06d807d50, {0x63c85b8, 0xc07bb0a600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc06d807d50, {0x63c85b8, 0xc07bb0a600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc07bb0a600}, {0x4f67ad2, 0x6446ff8}, 0xc0198a4fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219804494 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc05ca7c180}, 0xc030c3df98, 0x1, 0xc0751eb0a8, 0x519bbc, 0xc060fdd5c0, 0xc04cdc5800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05ca7c180}, 0x120, 0xc0751eb230, 0x60fdd560, 0x0, 0x660069, 0xc060fdd5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc05ca7c180}, 0xc060fdd560, {{{{0x0, 0x0}, {0xc04b2d0fc0, 0x56}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05ca7c180}, 0x6352410, 0x45bf59e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc05ca7c180}, {0xc060fdd440, {0x17430110306734ae, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc05ca7c180}, 0xc073d985a0, 0xc0751eb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05ca7c180}, 0x660089, {0xc060fdd440, {0x17430110306734ae, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc05ca7c180}, 0xc0839e6200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05ca7c0f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05ca7c0f0}, {0xc0751ed3f8, 0x660089}, 0xc0751ed400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05ca7c0f0}, {0x0}, 0xc0839e6000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05ca7c060}, 0xc0839e6000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}, 0xc08cfa7d60, 0xc02a063a28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}, 0x3, 0xc072696640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0751ed980}}}, {0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}, 0xc08cfa7d60, 0xc072696640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05ca7c060}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05ca7c060}, {0x40, 0xc0751edb70}, 0xc0751edab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05ca7c060}, {0x4e9d7a0, 0xc0839e6000}, 0xc08cfa7d60, 0xc02a063a28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05ca7c060}, 0xc0770627e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc060fdd320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc060fdd320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219150660 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bf9fe28, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc003531d80, 0xc063394000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc003531d80, {0xc063394000, 0x2b93, 0x2b93}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc003531d80, {0xc063394000, 0x2b93, 0xc0726e2dc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc025ab7c50, {0xc063394000, 0x7eda79, 0xc063394005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0796dbb60, {0xc063394000, 0x3, 0x2b93}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02a8dca98, {0xc063394000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03fd5e278, {0x631b900, 0xc02a8dca98}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03fd5e000, {0x7fe42ab59310, 0xc0796dbb60}, 0x2b93) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03fd5e000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03fd5e000, {0xc06c8ab000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc049760348, {0xc06c8ab000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc09b7a0088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc09b7a0088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc09b7a0660, {0x634aac0, 0xc09b7a0088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc09b7a0000, 0xc02856c860, 0xc07b4f542b, 0xc07b4f542d, {0x635bec0, 0xc05976ecf0}, 0xc000d6ae00, 0xc03b886528, 0xc07b4f5429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc09b7a0000, {0x63c85b8, 0xc0475fb7d0}, 0xc02856c850, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0541e0046, 0x8}}, 0x0, 0xc0475fb770, 0xc0475fb7a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0475fb7d0}, {0x643d810, 0xc0796dbb60}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0826507e0}, {0x643d810, 0xc0796dbb60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0826507e0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802511 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edab180, {0x63c85b8, 0xc067981dd0}, 0xc06ff4c5e8, 0x1, 0xc05141d0a8, 0x519bbc, 0xc03e9d67e0, 0xc0257c27e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc067981dd0}, 0x1e0, 0xc05141d230, 0x3e9d6780, 0x0, 0x660069, 0xc03e9d67e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edab180, {0x63c85b8, 0xc067981dd0}, 0xc03e9d6780, {{{{0x0, 0x0}, {0xc051d786c0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc067981dd0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701860, {0x63c85b8, 0xc067981dd0}, {0xc04d8b9b00, {0x174301100f7498e6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701860, {0x63c85b8, 0xc067981dd0}, 0xc06891aaa0, 0xc05141d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067981dd0}, 0x660089, {0xc04d8b9b00, {0x174301100f7498e6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ef1cc00, {0x63c85b8, 0xc067981dd0}, 0xc0aebea200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100f7498e6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100f7498e6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067981d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067981d40}, {0xc05141f3f8, 0x660089}, 0xc05141f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067981d40}, {0x7fe3317ed4b0}, 0xc0aebea000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067981c80}, 0xc0aebea000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}, 0xc086683d80, 0xc01e0e1200) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}, 0x3, 0xc036e3cfc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05141f980}}}, {0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}, 0xc086683d80, 0xc036e3cfc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc067981c80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067981c80}, {0x40, 0xc05141fb70}, 0xc05141fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc067981c80}, {0x4e9d7a0, 0xc0aebea000}, 0xc086683d80, 0xc01e0e1200) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc067981c80}, 0xc01c824ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04d8b99e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04d8b99e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801651 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fabc640, {0x63c85b8, 0xc071e82990}, 0xc03a23c738, 0x1, 0xc052d350a8, 0x519bbc, 0xc0652e2180, 0xc048e716e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc071e82990}, 0x120, 0xc052d35230, 0x652e2120, 0x0, 0x660069, 0xc0652e2180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fabc640, {0x63c85b8, 0xc071e82990}, 0xc0652e2120, {{{{0x0, 0x0}, {0xc05a536c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc071e82990}, 0x6352410, 0x76af6590) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02faba320, {0x63c85b8, 0xc071e82990}, {0xc0652e2000, {0x1743011019b32a63, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02faba320, {0x63c85b8, 0xc071e82990}, 0xc06ea1e5a0, 0xc052d359f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc071e82990}, 0x660089, {0xc0652e2000, {0x1743011019b32a63, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02facec00, {0x63c85b8, 0xc071e82990}, 0xc091605800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019b32a63, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019b32a63, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc071e82900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071e82900}, {0xc052d373f8, 0x660089}, 0xc052d37400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc071e82900}, {0x28}, 0xc091605600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc071e82870}, 0xc091605600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}, 0xc05914ff20, 0xc03bc45350) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}, 0x3, 0xc03e961080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc052d37980}}}, {0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}, 0xc05914ff20, 0xc03e961080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc071e82870}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071e82870}, {0x40, 0xc052d37b70}, 0xc052d37ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc071e82870}, {0x4e9d7a0, 0xc091605600}, 0xc05914ff20, 0xc03bc45350) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc071e82870}, 0xc02b438000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc064f07e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc064f07e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806658 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc0176d7170}, 0xc051eb1000, 0xc01c696a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0176d7170}, 0xc051eb1000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061c0e40f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061c0e40f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0176d70e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0176d70e0}, {0xc0820da7f8, 0x660089}, 0xc0820da800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0176d70e0}, {0xc00102e4e0}, 0xc051eb0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0176d7080}, 0xc051eb0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0119db760, {0x63c85b8, 0xc0176d6f30}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc04479e600, {0x63c85b8, 0xc0176d6f30}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc04479e600, {0x63c8510, 0xc03943c280}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc04479e600, {0x63c8510, 0xc03943c280}, {0x63f7db8, 0x9bf6f48}, 0xc0820df088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc04479e600, {0x63c85b8, 0xc057c49620}, {{0x63f7db8, 0x9bf6f48}, {0xc058a9570b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc04479e600, {0x63c85b8, 0xc057c49620}, {{0x63f7db8, 0x9bf6f48}, {0xc058a9570b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc058a9570b, 0x6}, 0x0, 0x0}, {0xb30d615, 0xedb7a8350, 0x0}, {0xb30db4a, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04479e600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04479e600, {0x63c8510, 0xc08a778540}, 0xc082d13dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc047727050) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc08a778540}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc047727050) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219794229 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x6d) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0bf6796f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc094a47180, {0xc0179119d0, 0x32, 0x6b}, {0xc02581f200, 0x32, 0x74}, 0x1, 0x96) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0bf679588, {0xc0179119d0, 0x1, 0x74}, {0xc02581f200, 0x28a, 0xc00deca790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0bf679588, {0xc0179119d0, 0x32, 0x6b}, {0xc02581f200, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0bf678cc0, {0xc02581f200, 0x74, 0x6b}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0bf678cc0, {0xc0179119d0, 0x74, 0xc02581f200}, {0xc02581f200, 0xc023fd4300, 0x31}, 0x76) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0bf678a00, {0xc02581f200, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc09c4482a8, {{0xc023fd4300, 0x31, 0x76}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc08ae25300, {{0xc06c3d6c00, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07347d800, {0x63c85b8, 0xc033bf6450}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc033bf6450}, {0x64a6560, 0xc08ae25300}, {0xc06c3d6c00, 0x3c, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc033bf6450}, {0x6466ed8, 0xc09c448000}, {0xc06c3d6c00, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc046ebe270}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01403db00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020011400, {0x63c85b8, 0xc033bf6450}, 0xc01403db00, 0xc053b579a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020011400, {0x63c85b8, 0xc033bf6450}, 0xc01403db00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc033bf63c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc033bf63c0}, {0xc0492c88f0, 0x660089}, 0xc0492c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc033bf63c0}, {0xc00102e4e0}, 0xc01403da00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc033bf6360}, 0xc01403da00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc033bf61b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804305 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0241e2cc0}, 0xc05b018750, 0x1, 0xc077f170a8, 0x519bbc, 0xc04d8dad80, 0xc04b267e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0241e2cc0}, 0x1e0, 0xc077f17230, 0x4d8dad20, 0x0, 0x660069, 0xc04d8dad80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0241e2cc0}, 0xc04d8dad20, {{{{0x0, 0x0}, {0xc04d732240, 0xef}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc052e2c060, {0x63c85b8, 0xc0241e2cc0}, 0xc07372ab90, 0x77f17428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0241e2cc0}, {0xc058a805a0, {0x1743011032101887, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0241e2cc0}, 0xc022d91720, 0xc077f179f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0241e2cc0}, 0x660089, {0xc058a805a0, {0x1743011032101887, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0241e2cc0}, 0xc083db2f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032101887, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032101887, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0241e2bd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0241e2bd0}, {0xc077f193f8, 0x660089}, 0xc077f19400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0241e2bd0}, {0x7fe3e0f55138}, 0xc083db2d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0241e2b40}, 0xc083db2d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}, 0xc03b2fe040, 0xc0d233a9a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}, 0x3, 0xc076580c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc077f19980}}}, {0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}, 0xc03b2fe040, 0xc076580c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0241e2b40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0241e2b40}, {0x40, 0xc077f19b70}, 0xc077f19ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0241e2b40}, {0x4e9d7a0, 0xc083db2d00}, 0xc03b2fe040, 0xc0d233a9a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0241e2b40}, 0xc0354738c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057c03440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057c03440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806875 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0929ab4a0}, 0xc084bbf068, 0x1, 0xc00429b0a8, 0x519bbc, 0xc065c03260, 0xc04f8dc720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0929ab4a0}, 0x120, 0xc00429b230, 0x65c03200, 0x0, 0x660069, 0xc065c03260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0929ab4a0}, 0xc065c03200, {{{{0x0, 0x0}, {0xc0831fd8c0, 0x112}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0929ab4a0}, 0x6352410, 0x1b928a90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0929ab4a0}, {0xc065c030e0, {0x1743011052feb154, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0929ab4a0}, 0xc047208b40, 0xc00429b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0929ab4a0}, 0x660089, {0xc065c030e0, {0x1743011052feb154, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0929ab4a0}, 0xc044a23f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011052feb154, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011052feb154, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0929ab410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0929ab410}, {0xc00429d3f8, 0x660089}, 0xc00429d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0929ab410}, {0x0}, 0xc044a23d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0929ab380}, 0xc044a23d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}, 0xc0907f93c0, 0xc04e68b278) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}, 0x3, 0xc08e0d1d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00429d980}}}, {0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}, 0xc0907f93c0, 0xc08e0d1d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0929ab380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0929ab380}, {0x40, 0xc00429db70}, 0xc00429dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0929ab380}, {0x4e9d7a0, 0xc044a23d00}, 0xc0907f93c0, 0xc04e68b278) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0929ab380}, 0xc0310a91a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc065c02fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc065c02fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808706 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc066df43f0}, 0xc07694e378, 0x1, 0xc00a7cd0a8, 0x519bbc, 0xc01c11e480, 0xc05b29a840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc072add800, {0x63c85b8, 0xc066df43f0}, 0xc05eb8fe60, 0xc00a7cd1a8, 0x1c11e480, 0x0, 0x2, 0xc01c11e480, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc066df43f0}, 0xc01c11e3c0, {{{{0xc059ed4480, 0x35}, {0xc0176b1440, 0x58}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x30, {0x63c85b8, 0xc066df43f0}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc066df43f0}, {0xc083f01440, {0x174301105ed28e8b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc066df43f0}, 0xc06d642820, 0xc00a7cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066df43f0}, 0x660089, {0xc083f01440, {0x174301105ed28e8b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc066df43f0}, 0xc057a0f000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105ed28e8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105ed28e8b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066df4360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066df4360}, {0xc00a7cf3f8, 0x660089}, 0xc00a7cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066df4360}, {0xc07d08ccd0}, 0xc057a0ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066df42d0}, 0xc057a0ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}, 0xc04fc87dc0, 0xc0310fb0c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}, 0x3, 0xc044148fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00a7cf980}}}, {0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}, 0xc04fc87dc0, 0xc044148fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066df42d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066df42d0}, {0x40, 0xc00a7cfb70}, 0xc00a7cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066df42d0}, {0x4e9d7a0, 0xc057a0ee00}, 0xc04fc87dc0, 0xc0310fb0c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066df42d0}, 0xc053f759e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc083f01200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc083f01200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809914 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc04604d620}, 0xc01b3eb7d0, 0x1, 0xc026dc70a8, 0x519bbc, 0xc084b02f00, 0xc08ac36060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04604d620}, 0x120, 0xc026dc7230, 0x84b02ea0, 0x0, 0x660069, 0xc084b02f00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc04604d620}, 0xc084b02ea0, {{{{0x0, 0x0}, {0xc01d95cb40, 0x4d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04604d620}, 0x6352410, 0x42a0fe58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc04604d620}, {0xc084b02d80, {0x1743011065f7b8c6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc04604d620}, 0xc0ce7f3cc0, 0xc026dc79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04604d620}, 0x660089, {0xc084b02d80, {0x1743011065f7b8c6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc04604d620}, 0xc08205ad00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065f7b8c6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065f7b8c6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04604d530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04604d530}, {0xc026dc93f8, 0x660089}, 0xc026dc9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04604d530}, {0xc02c6c0400}, 0xc08205ab00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04604d4a0}, 0xc08205ab00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}, 0xc025422960, 0xc02d533980) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}, 0x3, 0xc01e2fee40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026dc9980}}}, {0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}, 0xc025422960, 0xc01e2fee40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04604d4a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04604d4a0}, {0x40, 0xc026dc9b70}, 0xc026dc9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04604d4a0}, {0x4e9d7a0, 0xc08205ab00}, 0xc025422960, 0xc02d533980) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04604d4a0}, 0xc05f8c77a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc084b02c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc084b02c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802103 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0716f9400, {0x63c85b8, 0xc027c45ce0}, 0xc00fdb0d00, 0xc0686ef360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0716f9400, {0x63c85b8, 0xc027c45ce0}, 0xc00fdb0d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffd7e1284, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffd7e1284, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027c45c20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027c45c20}, {0xc00eb273f8, 0x660089}, 0xc00eb27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027c45c20}, {0x7fe3b672c188}, 0xc00fdb0a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027c457d0}, 0xc00fdb0a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}, 0xc06dbbeb00, 0xc05c4e9488) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}, 0x3, 0xc074980380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00eb27980}}}, {0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}, 0xc06dbbeb00, 0xc074980380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc027c457d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027c457d0}, {0x40, 0xc00eb27b70}, 0xc00eb27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc027c457d0}, {0x4e9d7a0, 0xc00fdb0a00}, 0xc06dbbeb00, 0xc05c4e9488) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc027c457d0}, 0xc0792b8840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c204d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c204d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806594 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea2b90, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc02dc1c980, 0xc08301c800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02dc1c980, {0xc08301c800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc02dc1c980, {0xc08301c800, 0x735, 0xc0c37b2140}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc06c099360, {0xc08301c800, 0x7eda79, 0xc08301c805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc054149e00, {0xc08301c800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc041138468, {0xc08301c800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc07901a978, {0x631b900, 0xc041138468}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc07901a700, {0x7fe42ab59310, 0xc054149e00}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc07901a700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc07901a700, {0xc087aca000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0171378d8, {0xc087aca000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04dd03588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04dd03588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04dd03b60, {0x634aac0, 0xc04dd03588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04dd03500, 0xc05aa488b0, 0xc06f0bb42b, 0xc06f0bb42d, {0x635bec0, 0xc08ab2e750}, 0xc000d6ae00, 0xc086433f98, 0xc06f0bb429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04dd03500, {0x63c85b8, 0xc08ced82d0}, 0xc05aa488a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc035a06037, 0x8}}, 0x0, 0xc08ced8270, 0xc08ced82a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc08ced82d0}, {0x643d810, 0xc054149e00}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc08aff76e0}, {0x643d810, 0xc054149e00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc08aff76e0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799514 [runnable]: sync.runtime_SemacquireMutex(0xc094a1e690, 0x8f, 0x7fdb949c0ec0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc094a1e538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc094a1e560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc08255c900, {0xffffffffffffff15, 0xc094a1e6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc08255c900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc08255c900, {0xc02d601c80, 0x32, 0x74}, {0xc0244fdb80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091c71998, {0xc02d601c80, 0x32, 0x74}, {0xc0244fdb80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091c702c0, {0xc0244fdb80, 0x74, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091c702c0, {0xc02d601c80, 0x74, 0xc0244fdb80}, {0xc0244fdb80, 0xc06d9a7680, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091c70000, {0xc0244fdb80, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc00db463a8, {{0xc06d9a7680, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc058b31f00, {{0xc0595d85a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f263800, {0x63c85b8, 0xc072bf7b00}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc072bf7b00}, {0x64a6560, 0xc058b31f00}, {0xc0595d85a0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc072bf7b00}, {0x6466ed8, 0xc00db46100}, {0xc0595d85a0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc015fb3a58}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc07350ef00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0205cac00, {0x63c85b8, 0xc072bf7b00}, 0xc07350ef00, 0xc067c08d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cac00, {0x63c85b8, 0xc072bf7b00}, 0xc07350ef00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072bf7a70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072bf7a70}, {0xc0ad8388f0, 0x660089}, 0xc0ad8388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072bf7a70}, {0xc00102e4e0}, 0xc07350ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072bf7a10}, 0xc07350ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc79320}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812506 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04ae6b680}, 0xc04784ab40, 0x1, 0xc00f0236d8, 0x519bbc, 0xc046050660, 0xc05af6e8a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc02da397a0, {0x63c85b8, 0xc04ae6b680}, 0xc073d50900, 0xc00f0237d8, 0x46050660, 0x0, 0x3, 0xc046050660, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04ae6b680}, 0xc0460505a0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0322d2000, 0xb3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ae6b680}, 0x6352260, 0x420133c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04ae6b680}, {0xc046050480, {0x174301105bff062f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04ae6b680}, 0xc04bbd1c20, 0xc00f024020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ae6b680}, 0x660089, {0xc046050480, {0x174301105bff062f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04ae6b680}, 0xc05707f000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ae6b5c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ae6b5c0}, {0xc07d279a28, 0x660089}, 0xc07d279a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ae6b5c0}, {0xc00102e4e0}, 0xc05707ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ae6b530}, 0xc05707ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0516581d0, {0x63c85b8, 0xc04ae6b4a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0516581d0, {0x63c85b8, 0xc04ae6b4a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc04ae6b4a0}, {0x4f67ad2, 0x10beec0}, 0xc034637fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219802696 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03005e780, {0x63c85b8, 0xc0cc6e2690}, 0xc01dbc1440, 0x1, 0xc0681850a8, 0x519bbc, 0xc040027920, 0xc077430cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0cc6e2690}, 0x120, 0xc068185230, 0x400278c0, 0x0, 0x660069, 0xc040027920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03005e780, {0x63c85b8, 0xc0cc6e2690}, 0xc0400278c0, {{{{0x0, 0x0}, {0xc024bbcb40, 0xa}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0cc6e2690}, 0x6352410, 0x14c740e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030007040, {0x63c85b8, 0xc0cc6e2690}, {0xc0400277a0, {0x174301102706574a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030007040, {0x63c85b8, 0xc0cc6e2690}, 0xc01c459d60, 0xc0681859f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0cc6e2690}, 0x660089, {0xc0400277a0, {0x174301102706574a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc0cc6e2690}, 0xc04c523300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102706574a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102706574a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0cc6e2600}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0cc6e2600}, {0xc0681873f8, 0x660089}, 0xc068187400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0cc6e2600}, {0x7461636f6c22202c}, 0xc04c523100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0cc6e2570}, 0xc04c523100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}, 0xc06798d0a0, 0xc018d92750) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}, 0x3, 0xc08d7a6e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc068187980}}}, {0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}, 0xc06798d0a0, 0xc08d7a6e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0cc6e2570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0cc6e2570}, {0x40, 0xc068187b70}, 0xc068187ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0cc6e2570}, {0x4e9d7a0, 0xc04c523100}, 0xc06798d0a0, 0xc018d92750) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0cc6e2570}, 0xc040ab3140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc064decfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc064decfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809915 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc023b91400, {0x63c85b8, 0xc04604df80}, 0xc02d533e90, 0x1, 0x70801805d1edfb, 0x37e11d600, 0xc084b035c0, 0xc081f8b5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04604df80}, 0x120, 0xc00f31b230, 0x84b03560, 0x0, 0x660069, 0xc084b035c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc023b91400, {0x63c85b8, 0xc04604df80}, 0xc084b03560, {{{{0x0, 0x0}, {0xc089989200, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04604df80}, 0x6352410, 0x42a0fe70) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc023a89450, {0x63c85b8, 0xc04604df80}, {0xc084b03440, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc023a89450, {0x63c85b8, 0xc04604df80}, 0xc06f966280, 0xc00f31b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04604df80}, 0x660089, {0xc084b03440, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023bef000, {0x63c85b8, 0xc04604df80}, 0xc08205b200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04604def0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04604def0}, {0xc00f31d3f8, 0x660089}, 0xc00f31d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04604def0}, {0xc0015221e0}, 0xc08205b000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04604de60}, 0xc08205b000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}, 0xc025422be0, 0xc02d533e60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}, 0x3, 0xc01e2ff2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00f31d980}}}, {0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}, 0xc025422be0, 0xc01e2ff2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04604de60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04604de60}, {0x40, 0xc00f31db70}, 0xc00f31dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04604de60}, {0x4e9d7a0, 0xc08205b000}, 0xc025422be0, 0xc02d533e60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04604de60}, 0xc05f8c7da0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc084b03320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc084b03320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810320 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc01c8796e0}, 0xc041f28240, 0x1, 0xc020b910a8, 0x519bbc, 0xc0751ae4e0, 0xc0578796e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01c8796e0}, 0x120, 0xc020b91230, 0x751ae480, 0x0, 0x660069, 0xc0751ae4e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc01c8796e0}, 0xc0751ae480, {{{{0x0, 0x0}, {0xc038958b40, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01c8796e0}, 0x6352410, 0x7818ed50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc01c8796e0}, {0xc0751ae360, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc01c8796e0}, 0xc0b208abe0, 0xc020b919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01c8796e0}, 0x660089, {0xc0751ae360, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc01c8796e0}, 0xc08cb12600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c879470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c879470}, {0xc020b933f8, 0x660089}, 0xc020b93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c879470}, {0x1fb000c71368eff}, 0xc08cb12400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c879380}, 0xc08cb12400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}, 0xc019fe3f40, 0xc0309cf878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}, 0x3, 0xc0768868c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc020b93980}}}, {0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}, 0xc019fe3f40, 0xc0768868c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01c879380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c879380}, {0x40, 0xc020b93b70}, 0xc020b93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01c879380}, {0x4e9d7a0, 0xc08cb12400}, 0xc019fe3f40, 0xc0309cf878) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01c879380}, 0xc030ceff80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0751ae240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0751ae240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803833 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01860c400, {0x63c85b8, 0xc0d1012ba0}, 0xc079bdc000, 0xc07839db80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01860c400, {0x63c85b8, 0xc0d1012ba0}, 0xc079bdc000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032120805, 0x0, 0x0}, 0xc02fd892b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032120805, 0x0, 0x0}, 0xc02fd892b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0d1012b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0d1012b10}, {0xc071ab13f8, 0x660089}, 0xc071ab1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0d1012b10}, {0x0}, 0xc072d73e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0d1012a80}, 0xc072d73e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}, 0xc03c05f6e0, 0xc0439b2be8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}, 0x3, 0xc0511dc580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc071ab1980}}}, {0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}, 0xc03c05f6e0, 0xc0511dc580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0d1012a80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0d1012a80}, {0x40, 0xc071ab1b70}, 0xc071ab1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0d1012a80}, {0x4e9d7a0, 0xc072d73e00}, 0xc03c05f6e0, 0xc0439b2be8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0d1012a80}, 0xc031f7cc60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802622 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e04d400, {0x63c85b8, 0xc02af30ea0}, 0xc0b5831900, 0xc081db0b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc02af30ea0}, 0xc0b5831900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffb8c5e20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffb8c5e20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02af30de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02af30de0}, {0xc04250f3f8, 0x660089}, 0xc04250f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02af30de0}, {0x0}, 0xc0b5831700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02af30b10}, 0xc0b5831700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}, 0xc0829bfa00, 0xc02010fe90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}, 0x3, 0xc037e9c0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04250f980}}}, {0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}, 0xc0829bfa00, 0xc037e9c0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02af30b10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02af30b10}, {0x40, 0xc04250fb70}, 0xc04250fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02af30b10}, {0x4e9d7a0, 0xc0b5831700}, 0xc0829bfa00, 0xc02010fe90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02af30b10}, 0xc073865140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c488c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c488c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219790315 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02a822fc0}, 0xc01a1b5cb0, {0x646cc48, 0xc02a8230e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02a822fc0}, 0xedb7a834e, 0xc05a62bd40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02a822fc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0204043c0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01a1b5ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0204043c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802326 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021c87400, {0x63c85b8, 0xc04cc062d0}, 0xc0487bd000, 0xc048083220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c87400, {0x63c85b8, 0xc04cc062d0}, 0xc0487bd000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102182df2a, 0x0, 0x0}, 0xc024e29b80, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102182df2a, 0x0, 0x0}, 0xc024e29b80, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04cc06240}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cc06240}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04cc06240}, {0xc00102e4e0}, 0xc0487bcf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04cc061e0}, 0xc0487bcf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc02e4eb5c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc076c7d860}, {0xc00dfd36b0, 0x24}, 0x46c6d96b8, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x4c1af3}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc076c7d860}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812273 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc03c8f3950}, 0xc076cdddd0, 0x1, 0xc0337790a8, 0x519bbc, 0xc064cbc420, 0xc04d339aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc07a61cc60, {0x63c85b8, 0xc03c8f3950}, 0x10, 0xc0337791a8, 0x1e, 0x20, 0x9183e40, 0xc064cbc420, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc03c8f3950}, 0xc064cbc3c0, {{{{0xc044876480, 0x10}, {0xc0aa8fe900, 0x27}}, {{0xc03a500b40, 0x13}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03c8f3950}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc03c8f3950}, {0x0, {0x1743011081250b0a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc03c8f3950}, 0xc04e572dc0, 0xc0337799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03c8f3950}, 0x660089, {0x0, {0x1743011081250b0a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc03c8f3950}, 0xc082d2e100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011081250b0a, 0x0, 0x0}, 0xc03ea9e390, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011081250b0a, 0x0, 0x0}, 0xc03ea9e390, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c8f38c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c8f38c0}, {0xc03377b3f8, 0x660089}, 0xc03377b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c8f38c0}, {0x7fe32e6d0278}, 0xc02751df00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c8f3830}, 0xc02751df00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}, 0xc031208b60, 0xc027bea0f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}, 0x3, 0xc045b51540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03377b980}}}, {0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}, 0xc031208b60, 0xc045b51540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c8f3830}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c8f3830}, {0x40, 0xc03377bb70}, 0xc03377bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c8f3830}, {0x4e9d7a0, 0xc02751df00}, 0xc031208b60, 0xc027bea0f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c8f3830}, 0xc042686f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04f42f7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04f42f7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219784739 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a5a210, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc079902080, 0xc07631c800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc079902080, {0xc07631c800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc079902080, {0xc07631c800, 0x739, 0xc0721afa40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc05591ca38, {0xc07631c800, 0x7eda79, 0xc07631c805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0cce342a0, {0xc07631c800, 0x46a86f, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc038a1aac8, {0xc07631c800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0788c1af8, {0x631b900, 0xc038a1aac8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0788c1880, {0x7fe42ab59310, 0xc0cce342a0}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0788c1880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0788c1880, {0xc0917e4000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc076d82390, {0xc0917e4000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc068382b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc068382b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0683830e0, {0x634aac0, 0xc068382b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc068382a80, 0xc076838780, 0xc093d0d42b, 0xc093d0d42d, {0x635bec0, 0xc074e2ce10}, 0xc000d6ae00, 0xc0763327b4, 0xc093d0d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc068382a80, {0x63c85b8, 0xc076047fb0}, 0xc076838770, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc090bc5043, 0xb}}, 0x0, 0xc076047f50, 0xc076047f80, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc076047fb0}, {0x643d810, 0xc0cce342a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc079f02e10}, {0x643d810, 0xc0cce342a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc079f02e10}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802668 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bab4000, {0x63c85b8, 0xc07274cf30}, 0xc0b0d49700, 0xc0798b5680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bab4000, {0x63c85b8, 0xc07274cf30}, 0xc0b0d49700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110256394b1, 0x0, 0x0}, 0xc0571b2fc0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110256394b1, 0x0, 0x0}, 0xc0571b2fc0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07274cea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07274cea0}, {0xc05ff453f8, 0x660089}, 0xc05ff45400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07274cea0}, {0x7fe13e89ce98}, 0xc0b0d49400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07274ce10}, 0xc0b0d49400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}, 0xc081f7eb00, 0xc02f67dcb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}, 0x3, 0xc064506240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05ff45980}}}, {0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}, 0xc081f7eb00, 0xc064506240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07274ce10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07274ce10}, {0x40, 0xc05ff45b70}, 0xc05ff45ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07274ce10}, {0x4e9d7a0, 0xc0b0d49400}, 0xc081f7eb00, 0xc02f67dcb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07274ce10}, 0xc03a1689c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0bcf738c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0bcf738c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804639 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc064821bc0}, 0xc066ab42e8, 0x1, 0xc0555ff0a8, 0x519bbc, 0xc0528c6240, 0xc04ba358c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc064821bc0}, 0x1e0, 0xc0555ff230, 0x528c61e0, 0x0, 0x660069, 0xc0528c6240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc064821bc0}, 0xc0528c61e0, {{{{0x0, 0x0}, {0xc01fe83440, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc064821bc0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc064821bc0}, {0xc06647b440, {0x17430110314418a7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc064821bc0}, 0xc04bbdcf00, 0xc0555ff9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc064821bc0}, 0x660089, {0xc06647b440, {0x17430110314418a7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc064821bc0}, 0xc08c835300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110314418a7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110314418a7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064821b30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064821b30}, {0xc0556013f8, 0x660089}, 0xc055601400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064821b30}, {0x7fe137b51c10}, 0xc08c835100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064821aa0}, 0xc08c835100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}, 0xc04a1d4d60, 0xc0664a2f30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}, 0x3, 0xc035095980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055601980}}}, {0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}, 0xc04a1d4d60, 0xc035095980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc064821aa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064821aa0}, {0x40, 0xc055601b70}, 0xc055601ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc064821aa0}, {0x4e9d7a0, 0xc08c835100}, 0xc04a1d4d60, 0xc0664a2f30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc064821aa0}, 0xc0add4f9e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06647b320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06647b320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219792489 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02d3a7e60}, 0xc07e11fcb0, {0x646cc48, 0xc02d3a7f80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02d3a7e60}, 0xedb7a834e, 0xc088161e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02d3a7e60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc047c39ad0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc07e11fef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc047c39ad0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219773179 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc03eeea490, 0x37) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc08ec6b500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc08ec6bb60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0b33ff900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0b33ff900, {0x63c8510, 0xc03eeea500}, 0xc0d1bd7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0415168d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03eeea500}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0415168d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808463 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc067e0faa0}, 0xc0653faf18, 0x1, 0x0, 0x1, 0xc0368d5320, 0xc01fd74060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc067e0faa0}, 0x1e0, 0xc0064a4728, 0x368d52c0, 0x0, 0x660069, 0xc0368d5320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc067e0faa0}, 0xc0368d52c0, {{{{0x0, 0x0}, {0xc040ffcfc0, 0x4e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc067e0faa0}, 0x6352410, 0x53d3d360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc067e0faa0}, {0xc05ab87b00, {0x174301105bff062f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc067e0faa0}, 0xc071304820, 0xc07ce42ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067e0faa0}, 0x660089, {0xc05ab87b00, {0x174301105bff062f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc067e0faa0}, 0xc015105c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067e0fa10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067e0fa10}, {0xc002fe68f0, 0x660089}, 0xc002fe68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067e0fa10}, {0xc00102e4e0}, 0xc015105b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067e0f9b0}, 0xc015105b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc067e0ee40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812782 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808484 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc034532b70}, 0xc082ab0c00, 0xc02ca6cdc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc034532b70}, 0xc082ab0c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066dd663a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066dd663a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc034532ae0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc034532ae0}, {0xc0791407f8, 0x660089}, 0xc079140800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc034532ae0}, {0xc00102e4e0}, 0xc082ab0b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc034532a50}, 0xc082ab0b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0296bab00, {0x63c85b8, 0xc0345327b0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc059a74c00, {0x63c85b8, 0xc0345327b0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc059a74c00, {0x63c8510, 0xc076eff800}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc059a74c00, {0x63c8510, 0xc076eff800}, {0x63f7db8, 0x9bf6f48}, 0xc079145088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc059a74c00, {0x63c85b8, 0xc07545b530}, {{0x63f7db8, 0x9bf6f48}, {0xc0a54ffd63, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc059a74c00, {0x63c85b8, 0xc07545b530}, {{0x63f7db8, 0x9bf6f48}, {0xc0a54ffd63, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0a54ffd63, 0x6}, 0x0, 0x0}, {0x105339fe, 0xedb7a8350, 0x0}, {0x10533e72, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc059a74c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc059a74c00, {0x63c8510, 0xc0212aab00}, 0xc075e55dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0540bf950) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0212aab00}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0540bf950) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802046 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0606fc540}, 0xc056cc7c80, 0x1, 0xc05fca50a8, 0x519bbc, 0xc024183aa0, 0xc03543ef60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0606fc540}, 0x1e0, 0xc05fca5230, 0x24183a40, 0x0, 0x660069, 0xc024183aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0606fc540}, 0xc024183a40, {{{{0x0, 0x0}, {0xc03dcef8c0, 0x39}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0606fc540}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0606fc540}, {0xc04b25efc0, {0x174301101cc965ab, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0606fc540}, 0xc046884be0, 0xc05fca59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0606fc540}, 0x660089, {0xc04b25efc0, {0x174301101cc965ab, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0606fc540}, 0xc052b23d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101cc965ab, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101cc965ab, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0606fc4b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0606fc4b0}, {0xc05fca73f8, 0x660089}, 0xc05fca7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0606fc4b0}, {0xc0015221e0}, 0xc052b23b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0606fc420}, 0xc052b23b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}, 0xc03c73adc0, 0xc06af89e18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}, 0x3, 0xc0cec29740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05fca7980}}}, {0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}, 0xc03c73adc0, 0xc0cec29740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0606fc420}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0606fc420}, {0x40, 0xc05fca7b70}, 0xc05fca7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0606fc420}, {0x4e9d7a0, 0xc052b23b00}, 0xc03c73adc0, 0xc06af89e18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0606fc420}, 0xc037facba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04b25eea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04b25eea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805760 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc01ea9c090}, 0xc08a7eac18, 0x1, 0xc0073770a8, 0x519bbc, 0xc037d27020, 0xc040eb8f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01ea9c090}, 0x120, 0xc007377230, 0x37d26fc0, 0x0, 0x660069, 0xc037d27020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc01ea9c090}, 0xc037d26fc0, {{{{0x0, 0x0}, {0xc07a811e60, 0x1d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01ea9c090}, 0x6352410, 0x45bf4518) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc01ea9c090}, {0xc037d26ea0, {0x174301104be59ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc01ea9c090}, 0xc03f2fa3c0, 0xc0073779f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01ea9c090}, 0x660089, {0xc037d26ea0, {0x174301104be59ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc01ea9c090}, 0xc053477c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104be59ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104be59ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01ea9c000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ea9c000}, {0xc0073793f8, 0x660089}, 0xc007379400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01ea9c000}, {0x39342d363938322d}, 0xc053477a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0769ddf50}, 0xc053477a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}, 0xc0915abe40, 0xc08a7eaf78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}, 0x3, 0xc093ea3980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007379980}}}, {0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}, 0xc0915abe40, 0xc093ea3980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0769ddf50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0769ddf50}, {0x40, 0xc007379b70}, 0xc007379ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0769ddf50}, {0x4e9d7a0, 0xc053477a00}, 0xc0915abe40, 0xc08a7eaf78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0769ddf50}, 0xc0274e5320, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037d26d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037d26d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812606 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc04f5ed8c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc040ae9000, 0xc055ec6e00, 0xc040ae9168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04f5ed8c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc055ec6e00, {0xc0a61500f0, 0x29, 0x50}, {0xc03338c0e0, 0x29, 0x6d}, 0x0, 0xb6) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0934bb538, {0xc0a61500f0, 0x1, 0x6d}, {0xc03338c0e0, 0xc0934ba500, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0934bb538, {0xc0a61500f0, 0x29, 0x50}, {0xc03338c0e0, 0x29, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0934ba7c0, {0xc03338c0e0, 0x0, 0xc040ae94e0}, 0x121ea30, 0x70) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0934ba7c0, {0xc0a61500f0, 0x29, 0x6d}, {0xc03338c0e0, 0x28, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0934ba500, {0xc03338c0e0, 0x29, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc03ebd5088, {{0xc06342fa40, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc057885200, {{0xc06342fa40, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc06dad2c00, {0x63c85b8, 0xc04f63b020}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04f63b020}, {0x64a6560, 0xc057885200}, {0xc06342fa40, 0x0, 0x30}, {0x17430110937b6e9d, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04f63b020}, {0x6466ed8, 0xc03ebd5080}, {0xc06342fa40, 0x28, 0x30}, {0x17430110937b6e9d, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0412822e8}, {{0x17430110937b6e9d, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110937b6e9d, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0455a2700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b148d39d, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0312b0400, {0x63c85b8, 0xc04f63b020}, 0xc0455a2700, 0xc05307e460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0312b0400, {0x63c85b8, 0xc04f63b020}, 0xc0455a2700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110937b6e9d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110937b6e9d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63ae70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63ae70}, {0xc040aef3f8, 0x660089}, 0xc040aef400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63ae70}, {0x794b547630474533}, 0xc0455a2500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63ade0}, 0xc0455a2500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}, 0xc03ebdd3c0, 0xc031cfae70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}, 0x3, 0xc033d33500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc040aef980}}}, {0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}, 0xc03ebdd3c0, 0xc033d33500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63ade0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63ade0}, {0x40, 0xc040aefb70}, 0xc040aefab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63ade0}, {0x4e9d7a0, 0xc0455a2500}, 0xc03ebdd3c0, 0xc031cfae70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63ade0}, 0xc02a97fe60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e0658c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e0658c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802463 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc02ecdccf0}, {0xc056a8e000, 0x2, 0xc09c0c7e58}, {0x0, {0x1743010fed76859a, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc02ecdccf0}, 0x18701, 0xc099162000, 0x2b, 0xc053859750) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc02ecdccf0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809408 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc02ba95590}, 0xc05b815700, 0xc052005040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc02ba95590}, 0xc05b815700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ff991e, 0x0, 0x0}, 0xc02fb9f4c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ff991e, 0x0, 0x0}, 0xc02fb9f4c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ba950e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ba950e0}, {0xc0084893f8, 0x660089}, 0xc008489400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ba950e0}, {0x7fe3ba654988}, 0xc05b815400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ba94f60}, 0xc05b815400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}, 0xc03acd4980, 0xc08b141e48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}, 0x3, 0xc075c97ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc008489980}}}, {0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}, 0xc03acd4980, 0xc075c97ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02ba94f60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ba94f60}, {0x40, 0xc008489b70}, 0xc008489ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02ba94f60}, {0x4e9d7a0, 0xc05b815400}, 0xc03acd4980, 0xc08b141e48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02ba94f60}, 0xc0502079e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc086e53d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc086e53d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809904 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ef68280, {0x63c85b8, 0xc0328875c0}, 0xc067f10270, 0x1, 0xc0269fd0a8, 0x519bbc, 0xc036393140, 0xc01ea2b8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0328875c0}, 0x1e0, 0xc0269fd230, 0x363930e0, 0x0, 0x660069, 0xc036393140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ef68280, {0x63c85b8, 0xc0328875c0}, 0xc0363930e0, {{{{0x0, 0x0}, {0xc046ed6b40, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0328875c0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01ef6c0a0, {0x63c85b8, 0xc0328875c0}, {0xc071b29d40, {0x1743011065f7b8c6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01ef6c0a0, {0x63c85b8, 0xc0328875c0}, 0xc078e86fa0, 0xc0269fd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0328875c0}, 0x660089, {0xc071b29d40, {0x1743011065f7b8c6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ef48800, {0x63c85b8, 0xc0328875c0}, 0xc01c6faa00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065f7b8c6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065f7b8c6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc032887530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc032887530}, {0xc0269ff3f8, 0x660089}, 0xc0269ff400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc032887530}, {0x408b27fc4123cf2b}, 0xc01c6fa700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0328874a0}, 0xc01c6fa700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}, 0xc05f82cbc0, 0xc067f11da0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}, 0x3, 0xc011f49e00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0269ff980}}}, {0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}, 0xc05f82cbc0, 0xc011f49e00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0328874a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0328874a0}, {0x40, 0xc0269ffb70}, 0xc0269ffab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0328874a0}, {0x4e9d7a0, 0xc01c6fa700}, 0xc05f82cbc0, 0xc067f11da0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0328874a0}, 0xc0459f0a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07aafa120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07aafa120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805331 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc016c4d000, {0x63c85b8, 0xc079153d70}, 0xc091597e00, 0xc02da58140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc079153d70}, 0xc091597e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079153ce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079153ce0}, {0xc07bed53f8, 0x660089}, 0xc07bed5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079153ce0}, {0x7fe3263784c0}, 0xc091597b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079153c50}, 0xc091597b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}, 0xc064a6c160, 0xc017750048) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}, 0x3, 0xc035d80400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07bed5980}}}, {0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}, 0xc064a6c160, 0xc035d80400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079153c50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079153c50}, {0x40, 0xc07bed5b06}, 0xc07bed5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}, 0x20300d, 0x20300d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079153c50}, {0x4e9d7a0, 0xc091597b00}, 0xc064a6c160, 0xc017750048) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079153c50}, 0xc0c4875500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a07b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a07b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807380 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021a38400, {0x63c85b8, 0xc0453de780}, 0xc09903bb00, 0xc083df4640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021a38400, {0x63c85b8, 0xc0453de780}, 0xc09903bb00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0453de6f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0453de6f0}, {0xc00e83f858, 0x660089}, 0xc00e83f860) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0453de6f0}, {0xc00102e4e0}, 0xc09903b900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0453de690}, 0xc09903b900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x1743011058c105b3, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc00f726a20}, 0x0, 0xc07a2cf600) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc00f726a20}, 0x1e) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxnRecover(0xc00d502dc0, {0x63c85b8, 0xc00f726a20}, 0xc04087c240, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:304 +0x135 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxn.func2({0x63c85b8, 0xc00f726a20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:160 +0x1df github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00f726a20}, {0x23, 0x30}, 0xc04fb1d6e0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).resolveIndeterminateCommitForTxn(0xc00d502dc0, 0xc04087c240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:133 +0x133 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery.(*manager).ResolveIndeterminateCommit.func1() github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnrecovery/manager.go:96 +0x25 github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).doCall(0xc00d502e40, 0xc03c8b14f0, {0xc053263f20, 0x24}, 0xc04fb1d790) github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:128 +0x3b created by github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).DoChan github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:121 +0x2f1 goroutine 219801202 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0302d3540, {0x63c85b8, 0xc024e61d70}, 0xc03fc3c300, 0x1, 0xc0470ff0a8, 0x519bbc, 0xc073076960, 0xc04a8c0840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc024e61d70}, 0x120, 0xc0470ff230, 0x73076900, 0x0, 0x660069, 0xc073076960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0302d3540, {0x63c85b8, 0xc024e61d70}, 0xc073076900, {{{{0x0, 0x0}, {0xc05a74cd80, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc024e61d70}, 0x6352410, 0x553a7de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301883c0, {0x63c85b8, 0xc024e61d70}, {0xc0730767e0, {0x174301101936f520, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301883c0, {0x63c85b8, 0xc024e61d70}, 0xc070f288c0, 0xc0470ff9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024e61d70}, 0x660089, {0xc0730767e0, {0x174301101936f520, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030309c00, {0x63c85b8, 0xc024e61d70}, 0xc0545f6700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101936f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101936f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024e61ce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024e61ce0}, {0xc0471013f8, 0x660089}, 0xc047101400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024e61ce0}, {0x0}, 0xc0545f6400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024e61bf0}, 0xc0545f6400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}, 0xc038cae660, 0xc028476048) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}, 0x3, 0xc0321f3d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc047101980}}}, {0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}, 0xc038cae660, 0xc0321f3d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc024e61bf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024e61bf0}, {0x40, 0xc047101b70}, 0xc047101ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc024e61bf0}, {0x4e9d7a0, 0xc0545f6400}, 0xc038cae660, 0xc028476048) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc024e61bf0}, 0xc02eb25740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0730766c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0730766c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812582 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805748 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc011da9800, {0x63c85b8, 0xc04261ce40}, 0xc074f27f00, 0xc081ccadc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc011da9800, {0x63c85b8, 0xc04261ce40}, 0xc074f27f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011047f3c080, 0x0, 0x0}, 0xc05ca1e9a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011047f3c080, 0x0, 0x0}, 0xc05ca1e9a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04261cdb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04261cdb0}, {0xc0044c93f8, 0x660089}, 0xc0044c9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04261cdb0}, {0x90733d111bee1e93}, 0xc074f27a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04261cd20}, 0xc074f27a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}, 0xc03f52f2e0, 0xc03c049608) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}, 0x3, 0xc0168457c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0044c9980}}}, {0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}, 0xc03f52f2e0, 0xc0168457c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04261cd20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04261cd20}, {0x40, 0xc0044c9b70}, 0xc0044c9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04261cd20}, {0x4e9d7a0, 0xc074f27a00}, 0xc03f52f2e0, 0xc03c049608) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04261cd20}, 0xc04e85a360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093732120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093732120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802524 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc02af314d0}, 0xc03f8d65e8, 0x1, 0xc04e56f0a8, 0x519bbc, 0xc0473736e0, 0xc037fe0180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02af314d0}, 0x120, 0xc04e56f230, 0x47373680, 0x0, 0x660069, 0xc0473736e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc02af314d0}, 0xc047373680, {{{{0x0, 0x0}, {0xc0473226c0, 0xb}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02af314d0}, 0x6352410, 0x473dc730) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc02af314d0}, {0xc047373560, {0x174301101c76aae4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc02af314d0}, 0xc0704d5b80, 0xc04e56f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02af314d0}, 0x660089, {0xc047373560, {0x174301101c76aae4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc02af314d0}, 0xc0b5831e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c76aae4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c76aae4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02af31440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02af31440}, {0xc04e5713f8, 0x660089}, 0xc04e571400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02af31440}, {0xc0015221e0}, 0xc0b5831c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02af31380}, 0xc0b5831c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}, 0xc0829bfb40, 0xc016a040c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}, 0x3, 0xc037e9c400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04e571980}}}, {0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}, 0xc0829bfb40, 0xc037e9c400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02af31380}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02af31380}, {0x40, 0xc04e571b70}, 0xc04e571ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02af31380}, {0x4e9d7a0, 0xc0b5831c00}, 0xc0829bfb40, 0xc016a040c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02af31380}, 0xc073865500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc047373440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc047373440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218904456 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc098f06fd0, {0x63c85b8, 0xc03c175170}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc02edaec00, {0x63c85b8, 0xc03c175170}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc02edaec00, {0x63c8510, 0xc055394380}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc02edaec00, {0x63c8510, 0xc055394380}, {0x63f7db8, 0x9bf6f48}, 0xc082555088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc02edaec00, {0x63c85b8, 0xc05094bec0}, {{0x63f7db8, 0x9bf6f48}, {0xc01a3d81ae, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc02edaec00, {0x63c85b8, 0xc05094bec0}, {{0x63f7db8, 0x9bf6f48}, {0xc01a3d81ae, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc01a3d81ae, 0x6}, 0x0, 0x0}, {0x188bd865, 0xedb7a8350, 0x0}, {0x188be11d, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc02edaec00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc02edaec00, {0x63c8510, 0xc0070428c0}, 0xc0d1813dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc084984270) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0070428c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc084984270) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812555 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc04f5ed8c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0157b6000, 0xc041e41168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04f5ed8c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0157b6000, {0xc03605f570, 0x29, 0x68}, {0xc03fcf4070, 0x29, 0x6d}, 0x0, 0xf6) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc09347f538, {0xc03605f570, 0x1, 0x6d}, {0xc03fcf4070, 0xc09347e500, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09347f538, {0xc03605f570, 0x29, 0x68}, {0xc03fcf4070, 0x29, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09347e7c0, {0xc03fcf4070, 0x0, 0xc041e414e0}, 0x121ea30, 0x10) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09347e7c0, {0xc03605f570, 0x29, 0x6d}, {0xc03fcf4070, 0x28, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09347e500, {0xc03fcf4070, 0x29, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0574c7b88, {{0xc03a992c60, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04fb25700, {{0xc03a992c60, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc049e20c00, {0x63c85b8, 0xc056614c90}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056614c90}, {0x64a6560, 0xc04fb25700}, {0xc03a992c60, 0x0, 0x30}, {0x1743011092d6d80b, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056614c90}, {0x6466ed8, 0xc0574c7b80}, {0xc03a992c60, 0x28, 0x30}, {0x1743011092d6d80b, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc037910948}, {{0x1743011092d6d80b, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011092d6d80b, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc083617d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b0a43d0b, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0312b0400, {0x63c85b8, 0xc056614c90}, 0xc083617d00, 0xc0794255e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0312b0400, {0x63c85b8, 0xc056614c90}, 0xc083617d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011092d6d80b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011092d6d80b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056614c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614c00}, {0xc041e473f8, 0x660089}, 0xc041e47400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056614c00}, {0xc0015221e0}, 0xc083617b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056614b70}, 0xc083617b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}, 0xc0ccf7fd40, 0xc038161db8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}, 0x3, 0xc071e54dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc041e47980}}}, {0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}, 0xc0ccf7fd40, 0xc071e54dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056614b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614b70}, {0x40, 0xc041e47b70}, 0xc041e47ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056614b70}, {0x4e9d7a0, 0xc083617b00}, 0xc0ccf7fd40, 0xc038161db8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056614b70}, 0xc071982e40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801004 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0302e3800, {0x63c85b8, 0xc0a8a8a300}, 0xc041afcf00, 0xc08ca4c0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0302e3800, {0x63c85b8, 0xc0a8a8a300}, 0xc041afcf00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100fab231d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100fab231d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0a8a8a270}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a8a8a270}, {0xc038cd33f8, 0x660089}, 0xc038cd3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0a8a8a270}, {0xc0015221e0}, 0xc041afcc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0a8a8a1b0}, 0xc041afcc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}, 0xc05c9fff00, 0xc05f9a2330) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}, 0x3, 0xc0392f8280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc038cd3980}}}, {0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}, 0xc05c9fff00, 0xc0392f8280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0a8a8a1b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a8a8a1b0}, {0x40, 0xc038cd3b70}, 0xc038cd3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0a8a8a1b0}, {0x4e9d7a0, 0xc041afcc00}, 0xc05c9fff00, 0xc05f9a2330) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0a8a8a1b0}, 0xc01f1acfc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04d749440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04d749440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219773178 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc059b86000, 0xc0761f91b8) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc059b86000, {0xc01c88ac80, 0x2e, 0x73}, {0xc01cfd2d80, 0x2e, 0x3a}, 0x0, 0xdb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092e6da88, {0xc01c88ac80, 0x1, 0x3a}, {0xc01cfd2d80, 0x56, 0xc0761f92f8}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092e6da88, {0xc01c88ac80, 0x2e, 0x73}, {0xc01cfd2d80, 0x2e, 0x3a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092e6d1c0, {0xc01cfd2d80, 0x2, 0x2c}, 0x2b, 0xc8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092e6d1c0, {0xc01c88ac80, 0x3a, 0xc094f5d5c0}, {0xc01cfd2d80, 0xc094f5d5c0, 0x2d}, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092e6cf00, {0xc01cfd2d80, 0x2e, 0x3a}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0ac0ee928, {{0xc094f5d5c0, 0x2d, 0xc0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc051cf0b00, {{0xc041e5c5f0, 0x26, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0408b4000, {0x63c85b8, 0xc0523d80c0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0523d80c0}, {0x64a6560, 0xc051cf0b00}, {0xc041e5c5f0, 0xc0761fa900, 0x50}, {0x174301109e649f11, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0523d80c0}, {0x6466ed8, 0xc0ac0ee680}, {0xc041e5c5f0, 0x26, 0x50}, {0x174301109e649f11, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02b876060}, {{0x174301109e649f11, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109e649f11, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04fb2d900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bc320411, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0160af000, {0x63c85b8, 0xc0523d80c0}, 0xc04fb2d900, 0xc085824000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0160af000, {0x63c85b8, 0xc0523d80c0}, 0xc04fb2d900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109e649f11, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xc, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109e649f11, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xc, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xc, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0523d8030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0523d8030}, {0xc0761ff458, 0x660089}, 0xc0761ff460) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0523d8030}, {0xc00102e4e0}, 0xc04fb2d700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc016a99fb0}, 0xc04fb2d700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xc, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00a2a34a0, {0x63c85b8, 0xc016a99e30}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00a2a34a0, {0x63c85b8, 0xc016a99e30}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc08e30d360, {0x63c85b8, 0xc016a99e30}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc084b48000, {0x63c85b8, 0xc016a99e30}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc043ed00a0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc04df40000) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc016a99d70) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/is_null_ops.eg.go:226 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc065baec00) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc05ffffef0, {0x63c8510, 0xc03a9bbfc0}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0d3e7a280, {0x63c8510, 0xc03a9bbfc0}, 0xc050a2cf20) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc03a9bbc40}, 0xc0748f7260, 0xc050a2cf20, 0xc04cdd7b00, 0xc0d0e56e00, 0xc03ea2e9e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc03a9bbc40}, 0xc03ea2e6f0, 0xc0748f7260, 0xc06e307140, {{0x63ca340, 0xc04cdd7a80}, 0x0}, 0xc0d0e56e00) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc03ea2e000, {0x63c8510, 0xc03a9bbc40}, 0xc03ea2e6f0, 0xc03a9bbc40, {0x7fe42b790cf0, 0xc06e307140}, 0x0, 0xc02423c6b8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc03ea2e000, {0x63c8510, 0xc03a9bbc40}, 0xc03ea2e6f0, {0x7fe42b790cf0, 0xc06e307140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc03ea2e000, {0x63c85b8, 0xc016a98ab0}, {{0x63f8d78, 0xc08c44d860}, {0xc08526400d, 0x4e}, 0x1, 0x1}, 0xc05f43cb40, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc03ea2e000, {0x63c85b8, 0xc016a98ab0}, {{0x63f8d78, 0xc08c44d860}, {0xc08526400d, 0x4e}, 0x1, 0x1}, 0xc05f43cb40, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc03ea2e000, {0x63c85b8, 0xc016a98ab0}, {0xc05f43cb40, {0xc0153e3a60, 0x1, 0x1}, {0xc03fac4828, 0x9, 0x9}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc09c0eb5a6, 0x0}, 0x0, {0xbf21d79, 0xedb7a8351, 0x0}, 0x1}, 0xc03ea2e000, 0xc076205980, 0xc076205970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03ea2e000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03ea2e000, {0x63c8510, 0xc03eeea300}, 0xc0709c1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0415167b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03eeea300}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0415167b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811308 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc07a25ad80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812779 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812568 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811258 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc07a25aa50}, 0xc03fdf5620, 0x1, 0xc0295890a8, 0x519bbc, 0xc08c3b3aa0, 0xc056dde0c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc052ca2e40, {0x63c85b8, 0xc07a25aa50}, 0xc067a5cd80, 0xc0295891a8, 0x21, 0x30, 0x0, 0xc08c3b3aa0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc07a25aa50}, 0xc08c3b3a40, {{{{0xc044876480, 0x10}, {0xc03bfa5b00, 0x25}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x30, {0x63c85b8, 0xc07a25aa50}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc07a25aa50}, {0xc05073d9e0, {0x17430110756b5c81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc07a25aa50}, 0xc070d93a40, 0xc0295899f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a25aa50}, 0x660089, {0xc05073d9e0, {0x17430110756b5c81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc07a25aa50}, 0xc06d136100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110756b5c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110756b5c81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a25a9c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a25a9c0}, {0xc02958b3f8, 0x660089}, 0xc02958b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a25a9c0}, {0x7fe3f72f6b30}, 0xc0862adf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a25a930}, 0xc0862adf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}, 0xc00cd979a0, 0xc00b73e5e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}, 0x3, 0xc0b1bf05c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02958b980}}}, {0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}, 0xc00cd979a0, 0xc0b1bf05c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a25a930}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a25a930}, {0x40, 0xc02958bb70}, 0xc02958bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a25a930}, {0x4e9d7a0, 0xc0862adf00}, 0xc00cd979a0, 0xc00b73e5e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a25a930}, 0xc0328184e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc049b88d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc049b88d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808377 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc044279e30}, 0xc052163878, 0x1, 0xc00426f0a8, 0x519bbc, 0xc01ad85500, 0xc04ec649c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc044279e30}, 0x1e0, 0xc00426f230, 0x1ad854a0, 0x0, 0x660069, 0xc01ad85500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc044279e30}, 0xc01ad854a0, {{{{0x0, 0x0}, {0xc055f09680, 0x121}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc044279e30}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc044279e30}, {0xc059d76240, {0x174301105b180f40, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc044279e30}, 0xc06cad06e0, 0xc00426f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044279e30}, 0x660089, {0xc059d76240, {0x174301105b180f40, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc044279e30}, 0xc06dcccc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b180f40, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b180f40, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044279da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044279da0}, {0xc0042713f8, 0x660089}, 0xc004271400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044279da0}, {0x7fe338171260}, 0xc06dccca00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044279d10}, 0xc06dccca00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}, 0xc0123148e0, 0xc0178be0c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}, 0x3, 0xc05ffa5780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004271980}}}, {0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}, 0xc0123148e0, 0xc05ffa5780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044279d10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044279d10}, {0x40, 0xc004271b70}, 0xc004271ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044279d10}, {0x4e9d7a0, 0xc06dccca00}, 0xc0123148e0, 0xc0178be0c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044279d10}, 0xc04154b800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ab56ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ab56ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810965 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc016233860}, 0xc0385c6960, 0x1, 0xc0229570a8, 0x519bbc, 0xc01a6c1e60, 0xc01a6c1500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc016233860}, 0x1e0, 0xc022957230, 0x1a6c1e00, 0x0, 0x660069, 0xc01a6c1e60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc016233860}, 0xc01a6c1e00, {{{{0x0, 0x0}, {0xc0141e2480, 0x63}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc04e610600, {0x63c85b8, 0xc016233860}, 0xc066baeb90, 0x22957428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc016233860}, {0xc07bf28000, {0x17430110701c76c4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc016233860}, 0xc04cf97220, 0xc0229579f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc016233860}, 0x660089, {0xc07bf28000, {0x17430110701c76c4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc016233860}, 0xc06304e700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110701c76c4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110701c76c4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0162334a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0162334a0}, {0xc0229593f8, 0x660089}, 0xc022959400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0162334a0}, {0x7fe3fb79c278}, 0xc04c48fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc016232de0}, 0xc04c48fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}, 0xc046910400, 0xc0267d47f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}, 0x3, 0xc025542040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc022959980}}}, {0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}, 0xc046910400, 0xc025542040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc016232de0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc016232de0}, {0x40, 0xc022959b70}, 0xc022959ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc016232de0}, {0x4e9d7a0, 0xc04c48fd00}, 0xc046910400, 0xc0267d47f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc016232de0}, 0xc03eb35200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06bc6de60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06bc6de60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794856 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc00fec7050) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc00fec70b8, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0c0319bf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc059245180, {0xc01bd3c880, 0x2b, 0x74}, {0xc011b5ae40, 0x2b, 0x36}, 0x1, 0x9b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0c0319a88, {0xc01bd3c880, 0x1, 0x36}, {0xc011b5ae40, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0c0319a88, {0xc01bd3c880, 0x2b, 0x74}, {0xc011b5ae40, 0x2b, 0x36}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0c03191c0, {0xc011b5ae40, 0x36, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0c03191c0, {0xc01bd3c880, 0x2b, 0xc011b5ae40}, {0xc011b5ae40, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0c0318f00, {0xc011b5ae40, 0x2b, 0x36}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc090b86688, {{0xc053f17ce0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc062b2b100, {{0xc053f17ce0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04188c400, {0x63c85b8, 0xc052285770}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc052285770}, {0x64a6560, 0xc062b2b100}, {0xc053f17ce0, 0x6302000010140000, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc052285770}, {0x6466ed8, 0xc090b86680}, {0xc053f17ce0, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c77ef78}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc056cc3f00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020af2800, {0x63c85b8, 0xc052285770}, 0xc056cc3f00, 0xc059bea1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020af2800, {0x63c85b8, 0xc052285770}, 0xc056cc3f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0522856e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0522856e0}, {0xc00fecd3f8, 0x660089}, 0xc00fecd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0522856e0}, {0xedb7a3418}, 0xc056cc3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc052285650}, 0xc056cc3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}, 0xc0b906d780, 0xc041b3db00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}, 0x3, 0xc0437a5800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00fecd980}}}, {0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}, 0xc0b906d780, 0xc0437a5800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc052285650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052285650}, {0x40, 0xc00fecdb70}, 0xc00fecdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc052285650}, {0x4e9d7a0, 0xc056cc3c00}, 0xc0b906d780, 0xc041b3db00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc052285650}, 0xc03e235260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04acebd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04acebd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805105 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02591d900, {0x63c85b8, 0xc0592a0120}, 0xc01fb0ba28, 0x1, 0x0, 0x1, 0xc0575c8ba0, 0xc046fae600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0592a0120}, 0x1e0, 0xc0973be728, 0x575c8b40, 0xc00c6ae000, 0x18, 0xc0575c8ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02591d900, {0x63c85b8, 0xc0592a0120}, 0xc0575c8b40, {{{{0x0, 0x0}, {0xc0613d26c0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x20300d, {0x63c85b8, 0xc0592a0120}, 0xc0c7792bc8, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0258d3270, {0x63c85b8, 0xc0592a0120}, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0258d3270, {0x63c85b8, 0xc0592a0120}, 0xc0459d8b40, 0xc076d8cee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0592a0120}, 0x660089, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025aab000, {0x63c85b8, 0xc0592a0120}, 0xc06173ab00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592a0090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592a0090}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592a0090}, {0xc00102e4e0}, 0xc06173aa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592a0030}, 0xc06173aa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fcdbec0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808241 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0634f66c0}, 0xc0479368a0, 0x1, 0xc0105710a8, 0x519bbc, 0xc06049afc0, 0xc061b8f740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc003e1e1e0, {0x63c85b8, 0xc0634f66c0}, 0xc039b74120, 0xc0105711a8, 0x6049afc0, 0x0, 0x2, 0xc06049afc0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0634f66c0}, 0xc06049af00, {{{{0xc059ed4480, 0x35}, {0xc026131d40, 0x54}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x30, {0x63c85b8, 0xc0634f66c0}, 0x0, 0x8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0634f66c0}, {0xc05ebc5b00, {0x1743010ff140ac6d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0634f66c0}, 0xc041b81860, 0xc0105719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0634f66c0}, 0x660089, {0xc05ebc5b00, {0x1743010ff140ac6d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0634f66c0}, 0xc05a87e200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff140ac6d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff140ac6d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0634f6630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0634f6630}, {0xc0105733f8, 0x660089}, 0xc010573400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0634f6630}, {0x0}, 0xc05a87e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0634f65a0}, 0xc05a87e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}, 0xc00ff18ea0, 0xc07e112d20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}, 0x3, 0xc0518a2d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010573980}}}, {0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}, 0xc00ff18ea0, 0xc0518a2d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0634f65a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0634f65a0}, {0x40, 0xc010573b70}, 0xc010573ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0634f65a0}, {0x4e9d7a0, 0xc05a87e000}, 0xc00ff18ea0, 0xc07e112d20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0634f65a0}, 0xc0c72bdb00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060adb560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060adb560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811642 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03bd19180, {0x63c85b8, 0xc051a21b00}, 0xc0433baf48, 0x1, 0xc0067c10a8, 0x519bbc, 0xc059127320, 0xc053d896e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc051a21b00}, 0x1e0, 0xc0067c1230, 0x591272c0, 0x0, 0x660069, 0xc059127320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03bd19180, {0x63c85b8, 0xc051a21b00}, 0xc0591272c0, {{{{0x0, 0x0}, {0xc00a8a79e0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc051a21b00}, 0x63521a0, 0x1baa32d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc067985680, {0x63c85b8, 0xc051a21b00}, {0xc04e62e240, {0x174301106f6cccce, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc067985680, {0x63c85b8, 0xc051a21b00}, 0xc049eddb80, 0xc0067c19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc051a21b00}, 0x660089, {0xc04e62e240, {0x174301106f6cccce, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc051a21b00}, 0xc0444e0600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f6cccce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f6cccce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc051a21a70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051a21a70}, {0xc0067c33f8, 0x660089}, 0xc0067c3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc051a21a70}, {0x7fe3ba666c98}, 0xc0444e0400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc051a219e0}, 0xc0444e0400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}, 0xc018346500, 0xc0433bb2c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}, 0x3, 0xc044803d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0067c3980}}}, {0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}, 0xc018346500, 0xc044803d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc051a219e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051a219e0}, {0x40, 0xc0067c3b70}, 0xc0067c3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc051a219e0}, {0x4e9d7a0, 0xc0444e0400}, 0xc018346500, 0xc0433bb2c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc051a219e0}, 0xc01b733140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc065ff3440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc065ff3440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801271 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ffbfb80, {0x63c85b8, 0xc0707e8900}, 0xc046e3e138, 0x1, 0xc00fcb90a8, 0x519bbc, 0xc01c292600, 0xc00ba6b920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0707e8900}, 0x120, 0xc00fcb9230, 0x1c2925a0, 0x0, 0x660069, 0xc01c292600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ffbfb80, {0x63c85b8, 0xc0707e8900}, 0xc01c2925a0, {{{{0x0, 0x0}, {0xc0774c5c20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0707e8900}, 0x6352410, 0x351a9480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f5537c0, {0x63c85b8, 0xc0707e8900}, {0xc0a20a58c0, {0x174301101ac4abf5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f5537c0, {0x63c85b8, 0xc0707e8900}, 0xc04d8a63c0, 0xc00fcb99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0707e8900}, 0x660089, {0xc0a20a58c0, {0x174301101ac4abf5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030109400, {0x63c85b8, 0xc0707e8900}, 0xc06549fe00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ac4abf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ac4abf5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc012093b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc012093b60}, {0xc00fcbb3f8, 0x660089}, 0xc00fcbb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc012093b60}, {0x7fe38e2ee2d8}, 0xc0af507e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc012093a10}, 0xc0af507e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}, 0xc0548c5a60, 0xc03f814b10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}, 0x3, 0xc0265dbd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00fcbb980}}}, {0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}, 0xc0548c5a60, 0xc0265dbd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc012093a10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc012093a10}, {0x40, 0xc00fcbbb70}, 0xc00fcbbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc012093a10}, {0x4e9d7a0, 0xc0af507e00}, 0xc0548c5a60, 0xc03f814b10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc012093a10}, 0xc01b5707e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0a20a57a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0a20a57a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219733077 [runnable]: sync.runtime_notifyListWait(0xc095ec74e0, 0x66) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc095ec7400) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:193 github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc095ec7400, {0x47e80c0, 0x9bf6f48}) github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:396 +0x479 runtime/pprof.Do({0x63c8548, 0xc000140068}, {{0xc000167060, 0xc000d3ab40, 0xc097200780}}, 0xc0972007c0) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/pebble/record.NewLogWriter.func2() github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:314 +0x5c created by github.com/cockroachdb/pebble/record.NewLogWriter github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:313 +0x329 goroutine 219804620 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fefda40, {0x63c85b8, 0xc093b41710}, 0xc041908108, 0x1, 0xc00384b0a8, 0x519bbc, 0xc09d1ab6e0, 0xc060e623c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc093b41710}, 0x120, 0xc00384b230, 0x9d1ab680, 0x0, 0x660069, 0xc09d1ab6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fefda40, {0x63c85b8, 0xc093b41710}, 0xc09d1ab680, {{{{0x0, 0x0}, {0xc04fa76480, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc093b41710}, 0x6352230, 0x52e78db8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea3a40, {0x63c85b8, 0xc093b41710}, {0xc09d1ab560, {0x1743011048ec4569, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea3a40, {0x63c85b8, 0xc093b41710}, 0xc063478780, 0xc00384b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc093b41710}, 0x660089, {0xc09d1ab560, {0x1743011048ec4569, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff55800, {0x63c85b8, 0xc093b41710}, 0xc052542d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011048ec4569, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011048ec4569, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc093b41680}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc093b41680}, {0xc00384d3f8, 0x660089}, 0xc00384d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc093b41680}, {0xc0015221e0}, 0xc052542b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc093b415f0}, 0xc052542b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}, 0xc089e9bda0, 0xc041908cf0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}, 0x3, 0xc035a1dc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00384d980}}}, {0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}, 0xc089e9bda0, 0xc035a1dc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc093b415f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc093b415f0}, {0x40, 0xc00384db70}, 0xc00384dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc093b415f0}, {0x4e9d7a0, 0xc052542b00}, 0xc089e9bda0, 0xc041908cf0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc093b415f0}, 0xc039456a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc09d1ab320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc09d1ab320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811046 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01f82e800, {0x63c85b8, 0xc03531a9c0}, 0xc07d06f000, 0xc041d42960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f82e800, {0x63c85b8, 0xc03531a9c0}, 0xc07d06f000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bf9e35a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bf9e35a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03531a930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03531a930}, {0xc0229d53f8, 0x660089}, 0xc0229d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03531a930}, {0x0}, 0xc07d06eb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03531a8a0}, 0xc07d06eb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}, 0xc01544a840, 0xc0a7c03a58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}, 0x3, 0xc05a669f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0229d5980}}}, {0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}, 0xc01544a840, 0xc05a669f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03531a8a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03531a8a0}, {0x40, 0xc0229d5b70}, 0xc0229d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03531a8a0}, {0x4e9d7a0, 0xc07d06eb00}, 0xc01544a840, 0xc0a7c03a58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03531a8a0}, 0xc01c739a40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc000f68ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc000f68ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809251 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01df528c0, {0x63c85b8, 0xc048cad9b0}, 0xc0157977e8, 0x1, 0xc01a9110a8, 0x519bbc, 0xc06c35dbc0, 0xc0ba6448a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0145f7200, {0x63c85b8, 0xc048cad9b0}, 0xc06c35dbc0, 0xc01a9111a8, 0x64, 0xa0, 0x0, 0xc06c35dbc0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01df528c0, {0x63c85b8, 0xc048cad9b0}, 0xc06c35db00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc01ce89200, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc048cad9b0}, 0x6352260, 0x603b8d28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dec4f00, {0x63c85b8, 0xc048cad9b0}, {0xc06c35d9e0, {0x1743011027f78ab5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dec4f00, {0x63c85b8, 0xc048cad9b0}, 0xc03f2fac80, 0xc01a9119f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc048cad9b0}, 0x660089, {0xc06c35d9e0, {0x1743011027f78ab5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01df61400, {0x63c85b8, 0xc048cad9b0}, 0xc055d20b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048cad920}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048cad920}, {0xc01a9133f8, 0x660089}, 0xc01a913400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048cad920}, {0xc0425ecb1f}, 0xc055d20800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048cad890}, 0xc055d20800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}, 0xc08a2aca20, 0xc03bb7c960) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}, 0x3, 0xc013648f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a913980}}}, {0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}, 0xc08a2aca20, 0xc013648f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048cad890}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048cad890}, {0x40, 0xc01a913b70}, 0xc01a913ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048cad890}, {0x4e9d7a0, 0xc055d20800}, 0xc08a2aca20, 0xc03bb7c960) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048cad890}, 0xc02ff9d980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06c35d8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06c35d8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806335 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e78ac80, {0x63c85b8, 0xc084ba1f80}, 0xc05fe673c8, 0x1, 0xc00c1bf0a8, 0x519bbc, 0xc0d2e69ec0, 0xc081efdec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc084ba1f80}, 0x120, 0xc00c1bf230, 0xd2e69e60, 0x0, 0x660069, 0xc0d2e69ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e78ac80, {0x63c85b8, 0xc084ba1f80}, 0xc0d2e69e60, {{{{0x0, 0x0}, {0xc090f2b0e0, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc084ba1f80}, 0x6352410, 0x6618b350) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0180e3bd0, {0x63c85b8, 0xc084ba1f80}, {0xc0d2e69d40, {0x174301104d82da47, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0180e3bd0, {0x63c85b8, 0xc084ba1f80}, 0xc04ff243c0, 0xc00c1bf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc084ba1f80}, 0x660089, {0xc0d2e69d40, {0x174301104d82da47, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e7d3000, {0x63c85b8, 0xc084ba1f80}, 0xc067617c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104d82da47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104d82da47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc084ba1ef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc084ba1ef0}, {0xc00c1c13f8, 0x660089}, 0xc00c1c1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc084ba1ef0}, {0x0}, 0xc067617a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc084ba1e60}, 0xc067617a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}, 0xc08623e6a0, 0xc01c98a258) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}, 0x3, 0xc07d5e1000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00c1c1980}}}, {0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}, 0xc08623e6a0, 0xc07d5e1000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc084ba1e60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc084ba1e60}, {0x40, 0xc00c1c1b06}, 0xc00c1c1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc084ba1e60}, {0x4e9d7a0, 0xc067617a00}, 0xc08623e6a0, 0xc01c98a258) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc084ba1e60}, 0xc027be91a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05aa5e7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05aa5e7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812850 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811292 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ecee280, {0x63c85b8, 0xc0772dd440}, 0xc0cdaf39b0, 0x1, 0x7080181344cdb5, 0x37e11d600, 0xc0d43ead80, 0xc06cb4cf60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0772dd440}, 0x1e0, 0xc031e9b230, 0xd43ead20, 0x0, 0x660069, 0xc0d43ead80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ecee280, {0x63c85b8, 0xc0772dd440}, 0xc0d43ead20, {{{{0x0, 0x0}, {0xc090f2ad80, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0772dd440}, 0x63521a0, 0x3c750460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ec810e0, {0x63c85b8, 0xc0772dd440}, {0xc05b980000, {0x1743011076693cdb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ec810e0, {0x63c85b8, 0xc0772dd440}, 0xc07783b5e0, 0xc031e9b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0772dd440}, 0x660089, {0xc05b980000, {0x1743011076693cdb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ecd9400, {0x63c85b8, 0xc0772dd440}, 0xc0be439b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011076693cdb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011076693cdb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0772dd3b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0772dd3b0}, {0xc031e9d3f8, 0x660089}, 0xc031e9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0772dd3b0}, {0x7fe137a9ce50}, 0xc0be439800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0772dd320}, 0xc0be439800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}, 0xc05ba78c20, 0xc0cdaf3980) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}, 0x3, 0xc05279b580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc031e9d980}}}, {0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}, 0xc05ba78c20, 0xc05279b580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0772dd320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0772dd320}, {0x40, 0xc031e9db70}, 0xc031e9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0772dd320}, {0x4e9d7a0, 0xc0be439800}, 0xc05ba78c20, 0xc0cdaf3980) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0772dd320}, 0xc04f6410e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc012c3de60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc012c3de60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801117 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02c4a8c00, {0x63c85b8, 0xc042ee33e0}, 0xc01fab2800, 0xc02d3f1b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc042ee33e0}, 0xc01fab2800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ae6e9bb, 0x0, 0x0}, 0xc09fbac180, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ae6e9bb, 0x0, 0x0}, 0xc09fbac180, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042ee3350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042ee3350}, {0xc052a333f8, 0x660089}, 0xc052a33400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042ee3350}, {0x27}, 0xc01fab2600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042ee32c0}, 0xc01fab2600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}, 0xc00a5ea960, 0xc023fc5158) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}, 0x3, 0xc0843229c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc052a33980}}}, {0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}, 0xc00a5ea960, 0xc0843229c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc042ee32c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042ee32c0}, {0x40, 0xc052a33b70}, 0xc052a33ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc042ee32c0}, {0x4e9d7a0, 0xc01fab2600}, 0xc00a5ea960, 0xc023fc5158) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc042ee32c0}, 0xc04312eea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0896ba5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0896ba5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800466 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc01b66bf20}, 0xc001f13cb0, {0x646cc48, 0xc01b66bf80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc01b66bf20}, 0xedb7a834e, 0xc034ef4900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc01b66bf20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0604edec0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc001f13ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0604edec0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804644 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc047989140}, 0xc06d612b40, 0x1, 0xc05ee230a8, 0x519bbc, 0xc06413f800, 0xc07b02dec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc047989140}, 0x120, 0xc05ee23230, 0x6413f7a0, 0x0, 0x660069, 0xc06413f800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc047989140}, 0xc06413f7a0, {{{{0x0, 0x0}, {0xc09298fd40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047989140}, 0x6352410, 0x421b1070) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc047989140}, {0xc06413f680, {0x1743011015406c30, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc047989140}, 0xc07345ed20, 0xc05ee239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047989140}, 0x660089, {0xc06413f680, {0x1743011015406c30, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc047989140}, 0xc065238c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011015406c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011015406c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0479890b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0479890b0}, {0xc05ee253f8, 0x660089}, 0xc05ee25400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0479890b0}, {0x7fe3f32e84a0}, 0xc065238a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047989020}, 0xc065238a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}, 0xc073e5a300, 0xc06f9b6ed0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}, 0x3, 0xc050531f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05ee25980}}}, {0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}, 0xc073e5a300, 0xc050531f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047989020}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047989020}, {0x40, 0xc05ee25b70}, 0xc05ee25ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047989020}, {0x4e9d7a0, 0xc065238a00}, 0xc073e5a300, 0xc06f9b6ed0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047989020}, 0xc06811a1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06413f560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06413f560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806510 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc08b93a150}, {0xc0597d86e0, 0x2, 0xc00dd60e58}, {0x0, {0x1743010ff1fd8407, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc08b93a150}, 0xd16a, 0xc0d43f17a0, 0x20, 0xc028740e30) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc08b93a150}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803641 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ff75c00, {0x63c85b8, 0xc045784c30}, 0xc064951300, 0xc05b6eb180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75c00, {0x63c85b8, 0xc045784c30}, 0xc064951300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fae3ac8, 0x0, 0x0}, 0xc052082590, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fae3ac8, 0x0, 0x0}, 0xc052082590, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045784ba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045784ba0}, {0xc06e3c73f8, 0x660089}, 0xc06e3c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045784ba0}, {0x7fe42574de20}, 0xc064951100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045784b10}, 0xc064951100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}, 0xc01bc5e260, 0xc06d6133f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}, 0x3, 0xc0d3fd84c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06e3c7980}}}, {0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}, 0xc01bc5e260, 0xc0d3fd84c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045784b10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045784b10}, {0x40, 0xc06e3c7b70}, 0xc06e3c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045784b10}, {0x4e9d7a0, 0xc064951100}, 0xc01bc5e260, 0xc06d6133f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045784b10}, 0xc01c66fce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc047774240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc047774240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806631 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc08d8ece40}, 0xc0b2e630c8, 0x1, 0xc002b170a8, 0x519bbc, 0xc0388482a0, 0xc00bee4f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08d8ece40}, 0x120, 0xc002b17230, 0x38848240, 0x0, 0x660069, 0xc0388482a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc08d8ece40}, 0xc038848240, {{{{0x0, 0x0}, {0xc093c1f8c0, 0x10d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08d8ece40}, 0x6352410, 0x17e2fcf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc08d8ece40}, {0xc038848120, {0x174301104f8bceda, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc08d8ece40}, 0xc025d9aaa0, 0xc002b179f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08d8ece40}, 0x660089, {0xc038848120, {0x174301104f8bceda, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc08d8ece40}, 0xc04a799a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104f8bceda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104f8bceda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08d8ecdb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d8ecdb0}, {0xc002b193f8, 0x660089}, 0xc002b19400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08d8ecdb0}, {0x5971717171717171}, 0xc04a799800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08d8ecd20}, 0xc04a799800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}, 0xc08e326960, 0xc09ca425b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}, 0x3, 0xc08a4eb600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002b19980}}}, {0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}, 0xc08e326960, 0xc08a4eb600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08d8ecd20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d8ecd20}, {0x40, 0xc002b19b70}, 0xc002b19ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08d8ecd20}, {0x4e9d7a0, 0xc04a799800}, 0xc08e326960, 0xc09ca425b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08d8ecd20}, 0xc0bd99a720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc038848000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc038848000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811497 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0b9dc82d0, {0x63c85b8, 0xc050e22000}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc050e22000}, {0x4f67ad2, 0x6446ff8}, 0xc001f12fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219806768 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc09074e720}, 0xc022671968, 0x1, 0xc0030230a8, 0x519bbc, 0xc06fac34a0, 0xc0d29f1a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc09074e720}, 0x120, 0xc003023230, 0x6fac3440, 0x0, 0x660069, 0xc06fac34a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc09074e720}, 0xc06fac3440, {{{{0x0, 0x0}, {0xc044af7440, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc09074e720}, 0x6352410, 0x23d18c48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc09074e720}, {0xc06fac3320, {0x174301104de41b45, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc09074e720}, 0xc020712be0, 0xc0030239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc09074e720}, 0x660089, {0xc06fac3320, {0x174301104de41b45, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc09074e720}, 0xc097d25d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104de41b45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104de41b45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09074e690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09074e690}, {0xc0030253f8, 0x660089}, 0xc003025400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09074e690}, {0x0}, 0xc097d25b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09074e600}, 0xc097d25b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}, 0xc09097ea80, 0xc01cb7e420) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}, 0x3, 0xc08cfa3a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003025980}}}, {0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}, 0xc09097ea80, 0xc08cfa3a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09074e600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09074e600}, {0x40, 0xc003025b70}, 0xc003025ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09074e600}, {0x4e9d7a0, 0xc097d25b00}, 0xc09097ea80, 0xc01cb7e420) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09074e600}, 0xc05c888a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06fac3200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06fac3200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802284 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc06714a270}, 0xc031af7008, 0x1, 0xc04f1e90a8, 0x519bbc, 0xc0398cef60, 0xc039e62060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06714a270}, 0x1e0, 0xc04f1e9230, 0x398cef00, 0x0, 0x660069, 0xc0398cef60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc06714a270}, 0xc0398cef00, {{{{0x0, 0x0}, {0xc02d27cd80, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06714a270}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc06714a270}, {0xc021351200, {0x1743011020ffad81, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc06714a270}, 0xc020c62280, 0xc04f1e99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06714a270}, 0x660089, {0xc021351200, {0x1743011020ffad81, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc06714a270}, 0xc04e8cad00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020ffad81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020ffad81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06714a1e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06714a1e0}, {0xc04f1eb3f8, 0x660089}, 0xc04f1eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06714a1e0}, {0x6964726f6f437a65}, 0xc04e8cab00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06714a150}, 0xc04e8cab00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}, 0xc08a6d18e0, 0xc0364c1aa0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}, 0x3, 0xc03f5e87c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04f1eb980}}}, {0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}, 0xc08a6d18e0, 0xc03f5e87c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06714a150}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06714a150}, {0x40, 0xc04f1ebb70}, 0xc04f1ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06714a150}, {0x4e9d7a0, 0xc04e8cab00}, 0xc08a6d18e0, 0xc0364c1aa0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06714a150}, 0xc028bee2a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc014e37d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc014e37d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219769933 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab98120, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc066ce8800, 0xc00bf83000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc066ce8800, {0xc00bf83000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc066ce8800, {0xc00bf83000, 0x735, 0xc05842ab40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0381d3c98, {0xc00bf83000, 0x7eda79, 0xc00bf83005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc08d0db3e0, {0xc00bf83000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04434fd70, {0xc00bf83000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0aea233f8, {0x631b900, 0xc04434fd70}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0aea23180, {0x7fe42ab59310, 0xc08d0db3e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0aea23180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0aea23180, {0xc01a0d8000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc044155ed8, {0xc01a0d8000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc07988cb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc07988cb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc07988d0e0, {0x634aac0, 0xc07988cb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc07988ca80, 0xc048b6bc50, 0xc09a2a342b, 0xc09a2a342d, {0x635bec0, 0xc045a24f30}, 0xc000d6ae00, 0xc0629a3cd8, 0xc09a2a3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc07988ca80, {0x63c85b8, 0xc05b7d7da0}, 0xc048b6bc40, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc05679301f, 0x8}}, 0x0, 0xc05b7d7d40, 0xc05b7d7d70, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc05b7d7da0}, {0x643d810, 0xc08d0db3e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06f3e9140}, {0x643d810, 0xc08d0db3e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06f3e9140}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809525 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc038b9b6e0}, 0xc023846558, 0x1, 0xc01b4610a8, 0x519bbc, 0xc0ccb4f6e0, 0xc05a760240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc038b9b6e0}, 0x1e0, 0xc01b461230, 0xccb4f680, 0x0, 0x660069, 0xc0ccb4f6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc038b9b6e0}, 0xc0ccb4f680, {{{{0x0, 0x0}, {0xc060ff7d40, 0xd9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc01b461448, {0x63c85b8, 0xc038b9b6e0}, 0x20301c, 0x20301c) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc038b9b6e0}, {0xc04d131680, {0x17430110656dcfdf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc038b9b6e0}, 0xc03059d040, 0xc01b4619f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc038b9b6e0}, 0x660089, {0xc04d131680, {0x17430110656dcfdf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc038b9b6e0}, 0xc072edfc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110656dcfdf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110656dcfdf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc038b9b650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc038b9b650}, {0xc01b4633f8, 0x660089}, 0xc01b463400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc038b9b650}, {0xc0015221e0}, 0xc072edfa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc038b9b5c0}, 0xc072edfa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}, 0xc024415360, 0xc034dc33f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}, 0x3, 0xc063aabb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01b463980}}}, {0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}, 0xc024415360, 0xc063aabb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc038b9b5c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc038b9b5c0}, {0x40, 0xc01b463b70}, 0xc01b463ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc038b9b5c0}, {0x4e9d7a0, 0xc072edfa00}, 0xc024415360, 0xc034dc33f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc038b9b5c0}, 0xc06da6ae40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04d131560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04d131560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805591 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a12c0, {0x63c85b8, 0xc036cc6060}, 0xc031fac228, 0x1, 0x708017e14436d5, 0x37e11d600, 0xc05f01d260, 0xc05950f260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc036cc6060}, 0x120, 0xc077fab230, 0x5f01d200, 0xc085fbad00, 0x18, 0xc05f01d260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a12c0, {0x63c85b8, 0xc036cc6060}, 0xc05f01d200, {{{{0x0, 0x0}, {0xc0486f9560, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc036cc6060}, 0x6352410, 0x71c3f658) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c4594f0, {0x63c85b8, 0xc036cc6060}, {0xc05f01d0e0, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c4594f0, {0x63c85b8, 0xc036cc6060}, 0xc0bde625a0, 0xc077fab9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc036cc6060}, 0x660089, {0xc05f01d0e0, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bdc9c00, {0x63c85b8, 0xc036cc6060}, 0xc01fb3b400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a7ada40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a7ada40}, {0xc077fad3f8, 0x660089}, 0xc077fad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a7ada40}, {0xc0015221e0}, 0xc01fb3b200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a7ad860}, 0xc01fb3b200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}, 0xc03a016660, 0xc031fac1e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}, 0x3, 0xc065b2f100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc077fad980}}}, {0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}, 0xc03a016660, 0xc065b2f100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a7ad860}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a7ad860}, {0x40, 0xc077fadb70}, 0xc077fadab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}, 0x203019, 0x203019) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a7ad860}, {0x4e9d7a0, 0xc01fb3b200}, 0xc03a016660, 0xc031fac1e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a7ad860}, 0xc06cde15c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ef2afc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ef2afc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219630158 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc04dd06e50, 0x24) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc01a735500) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc01a735b60) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc055930000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc055930000, {0x63c8510, 0xc04dd06e80}, 0xc090ac1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc060459b40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc04dd06e80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc060459b40) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812666 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806942 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f22b7c0, {0x63c85b8, 0xc021cf5aa0}, 0xc05b0dc888, 0x1, 0xc0051390a8, 0x519bbc, 0xc034eb9620, 0xc04b1ef080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc086d97b00, {0x63c85b8, 0xc021cf5aa0}, 0xc034eb9620, 0xc0051391a8, 0x31, 0x40, 0x0, 0xc034eb9620, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f22b7c0, {0x63c85b8, 0xc021cf5aa0}, 0xc034eb9560, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc00f103e60, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc021cf5aa0}, 0x6352260, 0x63fdd9c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f219d60, {0x63c85b8, 0xc021cf5aa0}, {0xc034eb9440, {0x1743011018bca093, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f219d60, {0x63c85b8, 0xc021cf5aa0}, 0xc06f8de780, 0xc0051399f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc021cf5aa0}, 0x660089, {0xc034eb9440, {0x1743011018bca093, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f6c7800, {0x63c85b8, 0xc021cf5aa0}, 0xc07ce06600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011018bca093, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011018bca093, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021cf58f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021cf58f0}, {0xc00513b3f8, 0x660089}, 0xc00513b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021cf58f0}, {0xc0015221e0}, 0xc07ce06400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021cf5200}, 0xc07ce06400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}, 0xc00230e240, 0xc0234b9800) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}, 0x3, 0xc07623a0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00513b980}}}, {0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}, 0xc00230e240, 0xc07623a0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc021cf5200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021cf5200}, {0x40, 0xc00513bb70}, 0xc00513bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc021cf5200}, {0x4e9d7a0, 0xc07ce06400}, 0xc00230e240, 0xc0234b9800) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc021cf5200}, 0xc05a5a8c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09a53d200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09a53d200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808562 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc042ef8bd0}, 0xc0d5736708, 0x1, 0xc0051df0a8, 0x519bbc, 0xc05ab0f5c0, 0xc059d775c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc042ef8bd0}, 0x120, 0xc0051df230, 0x5ab0f560, 0x0, 0x660069, 0xc05ab0f5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc042ef8bd0}, 0xc05ab0f560, {{{{0x0, 0x0}, {0xc093c1e240, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc042ef8bd0}, 0x6352410, 0x6bab2588) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc042ef8bd0}, {0xc05ab0f440, {0x174301105d4dee7b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc042ef8bd0}, 0xc0761ed680, 0xc0051df9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc042ef8bd0}, 0x660089, {0xc05ab0f440, {0x174301105d4dee7b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc042ef8bd0}, 0xc045536a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105d4dee7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105d4dee7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042ef8b40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042ef8b40}, {0xc0051e13f8, 0x660089}, 0xc0051e1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042ef8b40}, {0x408b27fc4123cf2b}, 0xc045536800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042ef8ab0}, 0xc045536800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}, 0xc07754c8a0, 0xc02c844078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}, 0x3, 0xc05bdeab80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0051e1980}}}, {0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}, 0xc07754c8a0, 0xc05bdeab80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc042ef8ab0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042ef8ab0}, {0x40, 0xc0051e1b70}, 0xc0051e1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc042ef8ab0}, {0x4e9d7a0, 0xc045536800}, 0xc07754c8a0, 0xc02c844078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc042ef8ab0}, 0xc026223d40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ab0f320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05ab0f320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806940 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0302d3540, {0x63c85b8, 0xc03af97170}, 0xc0202305b8, 0x1, 0x708017f1f7e4dd, 0x37e11d600, 0xc0718bdce0, 0xc064a87f20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0aca88480, {0x63c85b8, 0xc03af97170}, 0xc0718bdce0, 0xc0029611a8, 0x66, 0xa0, 0x0, 0xc0718bdce0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0302d3540, {0x63c85b8, 0xc03af97170}, 0xc0718bdc20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0212d8a20, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03af97170}, 0x6352260, 0x53414bc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301883c0, {0x63c85b8, 0xc03af97170}, {0xc0718bd9e0, {0x174301101936f520, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301883c0, {0x63c85b8, 0xc03af97170}, 0xc03ab4cbe0, 0xc0029619f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03af97170}, 0x660089, {0xc0718bd9e0, {0x174301101936f520, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030309c00, {0x63c85b8, 0xc03af97170}, 0xc07d41e300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101936f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101936f520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03af970e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03af970e0}, {0xc0029633f8, 0x660089}, 0xc002963400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03af970e0}, {0x3feffffe539c41e7}, 0xc07d41e100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03af96ff0}, 0xc07d41e100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}, 0xc090c20f40, 0xc020230588) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}, 0x3, 0xc090051a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002963980}}}, {0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}, 0xc090c20f40, 0xc090051a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03af96ff0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03af96ff0}, {0x40, 0xc002963b70}, 0xc002963ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03af96ff0}, {0x4e9d7a0, 0xc07d41e100}, 0xc090c20f40, 0xc020230588) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03af96ff0}, 0xc04c216ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc09a53cfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc09a53cfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803099 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc035f045d0}, 0xc0c11df170, 0x1, 0xc065a9d0a8, 0x519bbc, 0xc064b63c80, 0xc05fd16ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc035f045d0}, 0x120, 0xc065a9d230, 0x64b63c20, 0x0, 0x660069, 0xc064b63c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc035f045d0}, 0xc064b63c20, {{{{0x0, 0x0}, {0xc07cbafb00, 0xd4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc035f045d0}, 0x6352410, 0x3ad19ad8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc035f045d0}, {0xc064b63b00, {0x1743011026aefceb, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc035f045d0}, 0xc04eeaa780, 0xc065a9d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc035f045d0}, 0x660089, {0xc064b63b00, {0x1743011026aefceb, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc035f045d0}, 0xc043722a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011026aefceb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011026aefceb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc035f04540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc035f04540}, {0xc065a9f3f8, 0x660089}, 0xc065a9f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc035f04540}, {0x7fe3f326d4d0}, 0xc043722800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc035f044b0}, 0xc043722800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}, 0xc024d87e60, 0xc01f4b6a50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}, 0x3, 0xc020881440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc065a9f980}}}, {0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}, 0xc024d87e60, 0xc020881440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc035f044b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc035f044b0}, {0x40, 0xc065a9fb70}, 0xc065a9fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc035f044b0}, {0x4e9d7a0, 0xc043722800}, 0xc024d87e60, 0xc01f4b6a50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc035f044b0}, 0xc038e64c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc062f01320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc062f01320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802002 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc097f5c548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc097f5c5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0915f6cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc06308f500, {0xc01b4a7080, 0x2b, 0x30}, {0xc00d7b28c0, 0x2b, 0x6f}, 0x1, 0x6c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0915f6b88, {0xc01b4a7080, 0x1, 0x6f}, {0xc00d7b28c0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0915f6b88, {0xc01b4a7080, 0x2b, 0x30}, {0xc00d7b28c0, 0x2b, 0x6f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0915f62c0, {0xc00d7b28c0, 0x6f, 0x30}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0915f62c0, {0xc01b4a7080, 0x2b, 0xc00d7b28c0}, {0xc00d7b28c0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0915f6000, {0xc00d7b28c0, 0x2b, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05672f708, {{0xc06b6c27b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0936d2700, {{0xc06b6c27b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0484b0000, {0x63c85b8, 0xc06fc75080}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06fc75080}, {0x64a6560, 0xc0936d2700}, {0xc06b6c27b0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06fc75080}, {0x6466ed8, 0xc05672f700}, {0xc06b6c27b0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9aa08}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc064d48700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020eb8c00, {0x63c85b8, 0xc06fc75080}, 0xc064d48700, 0xc026869540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020eb8c00, {0x63c85b8, 0xc06fc75080}, 0xc064d48700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06fc74ff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fc74ff0}, {0xc0076c88f0, 0x660089}, 0xc0076c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06fc74ff0}, {0x24}, 0xc064d48600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fc74f90}, 0xc064d48600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0618cb170}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219751188 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0296bad10, {0x63c85b8, 0xc0720b9d10}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0ce8c5300, {0x63c85b8, 0xc0720b9d10}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0ce8c5300, {0x63c8510, 0xc04ad146c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc0ce8c5300, {0x63c8510, 0xc04ad146c0}, {0x63f7db8, 0x9bf6f48}, 0xc0832ed088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0ce8c5300, {0x63c85b8, 0xc06fd1aea0}, {{0x63f7db8, 0x9bf6f48}, {0xc014f74a1b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0ce8c5300, {0x63c85b8, 0xc06fd1aea0}, {{0x63f7db8, 0x9bf6f48}, {0xc014f74a1b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc014f74a1b, 0x6}, 0x0, 0x0}, {0x10637ce9, 0xedb7a8350, 0x0}, {0x106383bc, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0ce8c5300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0ce8c5300, {0x63c8510, 0xc0226fcb80}, 0xc06dd95dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03c7a69c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0226fcb80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03c7a69c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811915 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0293b78f0}, 0xc051b45878, 0x1, 0xc0337f50a8, 0x519bbc, 0xc03c924060, 0xc025830ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0293b78f0}, 0x1e0, 0xc0337f5230, 0x3c924000, 0x0, 0x660069, 0xc03c924060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0293b78f0}, 0xc03c924000, {{{{0x0, 0x0}, {0xc093c9bd40, 0x121}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0293b78f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0293b78f0}, {0xc057defd40, {0x1743011078eab475, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0293b78f0}, 0xc02b18b7c0, 0xc0337f59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0293b78f0}, 0x660089, {0xc057defd40, {0x1743011078eab475, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0293b78f0}, 0xc01f6ff300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011078eab475, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011078eab475, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0293b7830}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0293b7830}, {0xc0337f73f8, 0x660089}, 0xc0337f7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0293b7830}, {0x0}, 0xc01f6ff100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0293b77a0}, 0xc01f6ff100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}, 0xc0bb7ebc20, 0xc051b45e00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}, 0x3, 0xc035ac6740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0337f7980}}}, {0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}, 0xc0bb7ebc20, 0xc035ac6740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0293b77a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0293b77a0}, {0x40, 0xc0337f7b70}, 0xc0337f7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0293b77a0}, {0x4e9d7a0, 0xc01f6ff100}, 0xc0bb7ebc20, 0xc051b45e00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0293b77a0}, 0xc02fc356e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc057defc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc057defc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801434 [runnable]: sync.runtime_SemacquireMutex(0xc097a62690, 0x8f, 0x7fe325102fc0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x3, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafb8, 0xc097a62560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc084756480, {0xffffffffffffff15, 0xc097a626c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc084756480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc084756480, {0xc042a26b60, 0x32, 0x64}, {0xc02db19180, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc079aee898, {0xc042a26b60, 0x32, 0x64}, {0xc02db19180, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc079aed1c0, {0xc02db19180, 0x74, 0x64}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc079aed1c0, {0xc042a26b60, 0x74, 0xc02db19180}, {0xc02db19180, 0xc099f07d90, 0x31}, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc079aecf00, {0xc02db19180, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc017178828, {{0xc099f07d90, 0x31, 0xb0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc007403700, {{0xc060159cb0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0c11a7000, {0x63c85b8, 0xc02c5519e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02c5519e0}, {0x64a6560, 0xc007403700}, {0xc060159cb0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02c5519e0}, {0x6466ed8, 0xc017178580}, {0xc060159cb0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc03d3af830}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc044b5ef00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c94800, {0x63c85b8, 0xc02c5519e0}, 0xc044b5ef00, 0xc0ad172b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c94800, {0x63c85b8, 0xc02c5519e0}, 0xc044b5ef00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02c551770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c551770}, {0xc0d68368f0, 0x660089}, 0xc0d68368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02c551770}, {0xc00102e4e0}, 0xc044b5ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02c551710}, 0xc044b5ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02ef02510}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812214 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020433c00, {0x63c85b8, 0xc030190930}, 0xc048c45b00, 0xc083df4c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020433c00, {0x63c85b8, 0xc030190930}, 0xc048c45b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f0ff4ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f0ff4ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc030190690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030190690}, {0xc03973d3f8, 0x660089}, 0xc03973d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc030190690}, {0x466d4d336b6a5a35}, 0xc048c45800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc030190600}, 0xc048c45800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}, 0xc01e5176a0, 0xc0c05c2498) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}, 0x3, 0xc03d2c8680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03973d980}}}, {0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}, 0xc01e5176a0, 0xc03d2c8680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc030190600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030190600}, {0x40, 0xc03973db70}, 0xc03973dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc030190600}, {0x4e9d7a0, 0xc048c45800}, 0xc01e5176a0, 0xc0c05c2498) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc030190600}, 0xc0c71f42a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc01e3d7560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc01e3d7560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808165 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ff75c00, {0x63c85b8, 0xc03fbe90e0}, 0xc02a7d6400, 0xc03b364140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75c00, {0x63c85b8, 0xc03fbe90e0}, 0xc02a7d6400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aaf3ead, 0x0, 0x0}, 0xc04f551610, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aaf3ead, 0x0, 0x0}, 0xc04f551610, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03fbe9050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fbe9050}, {0xc00eb3f3f8, 0x660089}, 0xc00eb3f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03fbe9050}, {0x7a4e78597a4e3245}, 0xc02a7d6000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03fbe8fc0}, 0xc02a7d6000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}, 0xc071b568e0, 0xc031108060) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}, 0x3, 0xc07013fb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00eb3f980}}}, {0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}, 0xc071b568e0, 0xc07013fb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03fbe8fc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03fbe8fc0}, {0x40, 0xc00eb3fb70}, 0xc00eb3fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03fbe8fc0}, {0x4e9d7a0, 0xc02a7d6000}, 0xc071b568e0, 0xc031108060) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03fbe8fc0}, 0xc074175920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07614b9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07614b9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807712 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ff32000, {0x63c85b8, 0xc01fee5050}, 0xc09cc38700, 0xc074334820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff32000, {0x63c85b8, 0xc01fee5050}, 0xc09cc38700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105914104d, 0x0, 0x0}, 0xc090c01c60, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105914104d, 0x0, 0x0}, 0xc090c01c60, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fee4fc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fee4fc0}, {0xc00753b3f8, 0x660089}, 0xc00753b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fee4fc0}, {0x40e8faff2ddd4c3a}, 0xc09cc38500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fee4f30}, 0xc09cc38500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}, 0xc016e3ad20, 0xc0765df4b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}, 0x3, 0xc05fde9b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00753b980}}}, {0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}, 0xc016e3ad20, 0xc05fde9b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01fee4f30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fee4f30}, {0x40, 0xc00753bb70}, 0xc00753bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01fee4f30}, {0x4e9d7a0, 0xc09cc38500}, 0xc016e3ad20, 0xc0765df4b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01fee4f30}, 0xc0610271a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc022e517a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc022e517a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810088 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc048e073e0}, 0xc01526acb0, {0x646cc48, 0xc048e07440}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc048e073e0}, 0xedb7a8350, 0xc09fae0480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc048e073e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06c14e8d0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01526aef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06c14e8d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803645 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a59c70, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0667f7400, 0xc04c6b5800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0667f7400, {0xc04c6b5800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0667f7400, {0xc04c6b5800, 0x0, 0x4d889e}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc041bc5658, {0xc04c6b5800, 0x18b71da, 0x4c6b580}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc058086060, {0xc04c6b5800, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc06613df38, {0xc04c6b5800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04d8a2cf8, {0x631b900, 0xc06613df38}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04d8a2a80, {0x7fe42ab59310, 0xc058086060}, 0x63c85b8) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04d8a2a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04d8a2a80, {0xc0b1c44000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0385c7818, {0xc0b1c44000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc049b65588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc049b65588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc049b65b60, {0x634aac0, 0xc049b65588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc049b65500, 0xc01dc59de0, 0xc09c57b42b, 0xc09c57b42d, {0x635bec0, 0xc011ac6090}, 0xc000d6ae00, 0xc061d8a7d8, 0xc09c57b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc049b65500, {0x63c85b8, 0xc02cfbdf50}, 0xc01dc59dd0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09f30903b, 0x8}}, 0x0, 0xc02cfbdef0, 0xc02cfbdf20, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02cfbdf50}, {0x643d810, 0xc058086060}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01f6ef440}, {0x643d810, 0xc058086060}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01f6ef440}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219756789 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0c8b7c580, {0x63c85b8, 0xc0211319b0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0528b8c00, {0x63c85b8, 0xc0211319b0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0528b8c00, {0x63c8510, 0xc0cc0d3580}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc0528b8c00, {0x63c8510, 0xc0cc0d3580}, {0x63f7db8, 0x9bf6f48}, 0xc095039088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0528b8c00, {0x63c85b8, 0xc0682a8600}, {{0x63f7db8, 0x9bf6f48}, {0xc04daee11d, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0528b8c00, {0x63c85b8, 0xc0682a8600}, {{0x63f7db8, 0x9bf6f48}, {0xc04daee11d, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc04daee11d, 0x6}, 0x0, 0x0}, {0x1e3b4bf5, 0xedb7a8350, 0x0}, {0x1e3b5646, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0528b8c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0528b8c00, {0x63c8510, 0xc062094e40}, 0xc070133dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a69aa00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc062094e40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a69aa00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219807114 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fc5cc80, {0x63c85b8, 0xc04a8de000}, 0xc05294a0f0, 0x1, 0xc003cb30a8, 0x519bbc, 0xc082ccf800, 0xc06d8c7740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc07c654a20, {0x63c85b8, 0xc04a8de000}, 0xc082ccf800, 0xc003cb31a8, 0x33, 0x40, 0x0, 0xc082ccf800, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fc5cc80, {0x63c85b8, 0xc04a8de000}, 0xc0665307e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc03936fd40, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04a8de000}, 0x63521a0, 0x321a13f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fc6a500, {0x63c85b8, 0xc04a8de000}, {0xc08e351320, {0x1743010ffecca49d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fc6a500, {0x63c85b8, 0xc04a8de000}, 0xc02c0a1360, 0xc003cb39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04a8de000}, 0x660089, {0xc08e351320, {0x1743010ffecca49d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fc83800, {0x63c85b8, 0xc04a8de000}, 0xc08fd19700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffecca49d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffecca49d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e479f50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e479f50}, {0xc003cb53f8, 0x660089}, 0xc003cb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e479f50}, {0x3feffffffd645321}, 0xc08fd19400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e479e30}, 0xc08fd19400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}, 0xc077d4cc80, 0xc06d5fa3a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}, 0x3, 0xc0612c5e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003cb5980}}}, {0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}, 0xc077d4cc80, 0xc0612c5e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e479e30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e479e30}, {0x40, 0xc003cb5b70}, 0xc003cb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e479e30}, {0x4e9d7a0, 0xc08fd19400}, 0xc077d4cc80, 0xc06d5fa3a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e479e30}, 0xc066530660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08e350d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08e350d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804622 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc092b56060}, 0xc041909098, 0x1, 0x0, 0x1, 0xc09ce383c0, 0xc05477a180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc092b56060}, 0x120, 0xc003a8a728, 0x9ce38360, 0x0, 0x660069, 0xc09ce383c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc092b56060}, 0xc09ce38360, {{{{0x0, 0x0}, {0xc04b734ea0, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc092b56060}, 0x6352410, 0x52e78dd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc092b56060}, {0xc09d1abe60, {0x17430110485ecb95, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc092b56060}, 0xc085cb6000, 0xc082d14ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc092b56060}, 0x660089, {0xc09d1abe60, {0x17430110485ecb95, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc092b56060}, 0xc052543300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110485ecb95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110485ecb95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc093b41fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc093b41fb0}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc093b41fb0}, {0xc00102e4e0}, 0xc052543200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc093b41f50}, 0xc052543200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc093b41c20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812425 [semacquire]: sync.runtime_SemacquireMutex(0xc097512b90, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc097512be0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc097512b90, 0x42, 0x50, 0xc097512be0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x4d866e, 0x9185600}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).init(0xc0446b2d80, 0xc00e0b2580, {0xc097512b90, 0x42, 0x50}, {0xc097512be0, 0x43, 0x50}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1229 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00e0b2580, {0xc097512b90, 0x42, 0x50}, {0xc097512be0, 0x43, 0x50}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2157 +0xdb github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a528c0, 0x43, 0xc09341d9e8, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0xc06290a1c0, 0xc09341db00, 0x0, 0xc09341da00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc09341d998, 0xc0734106e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc09341d998, {0xc01194f8b0, 0x42, 0x42}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:400 +0x97 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09341c2c0, {0xc01194f8b0, 0x50, 0x42}, 0x43, 0x42) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc09341c2c0, {0xc01194f8b0, 0xc01194f8b0, 0xc097512be0}, 0x42) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc09341c000, {0xc01194f8b0, 0x42, 0x42}, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/pebble.(*Iterator).SeekGE(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:865 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0910ee178, {{0xc07a12c0c0, 0x41, 0x60}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:290 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc03e747c00, {{0xc07a12c0c0, 0x41, 0x60}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.mvccGetMetadata({0x64a6560, 0xc03e747c00}, {{0xc07a12c0c0, 0x41, 0x60}, {0x0, 0x0, 0x0}}, 0x0, 0xc037ec4480) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:856 +0xbf github.com/cockroachdb/cockroach/pkg/storage.MVCCGarbageCollect({0x63c85b8, 0xc04f5699b0}, {0x7fe42e217a50, 0xc05ed95740}, 0xc018b84000, {0xc04f569a10, 0x2700e700000000, 0x1}, {0x174301109e3eaad1, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:3491 +0x59a github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.GC({_, _}, {_, _}, {{0x64d2498, 0xc02e0b2c00}, {{0x174301109e3eaad1, 0x0, 0x0}, 0xc033eafa20, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_gc.go:115 +0x305 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109e3eaad1, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:493 +0x287 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01f6a2000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110bc0c0fd1, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:644 +0x190 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110bc0c0fd1, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:612 +0x2da github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, {{0x17430110bc0c0fd1, 0x0, 0x0}, {0x174301109e3eaad1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:428 +0x5f3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc02e0b2c00, {0x63c85b8, 0xc04f5699b0}, {0xc033eafa30, 0x8}, 0xc01f6a2000, {{0x17430110bc0c0fd1, 0x0, 0x0}, {0x174301109e3eaad1, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:673 +0x1c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x174301109e3eaad1, {0x63c85b8, _}, {_, _}, _, {{{0x1742da059a8832ca, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:773 +0xaa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc02e0b2c00, {0x63c85b8, _}, _, _, {{{0x1742da059a8832ca, 0x0, 0x0}, 0x0, {0x1f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:113 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0b2c00, {0x63c85b8, 0xc04f5699b0}, 0xc01f6a2000, 0xc06cf4f220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:167 +0x7b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc04f5699b0}, 0xc01f6a2000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109e3eaad1, 0x0, 0x0}, 0xc033eafa20, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109e3eaad1, 0x0, 0x0}, 0xc033eafa20, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f5698c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f5698c0}, {0xc07d93f918, 0x660089}, 0xc07d93f920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f5698c0}, {0xc00102e4e0}, 0xc070375f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f569860}, 0xc070375f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06123b740}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04f5693e0}, {0xc00dfd36b0, 0x24}, 0x4608cc2f6, 0xc02790dd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc057b313e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04f5693e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810830 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0430ca8a0}, 0xc03c2efb60, 0x1, 0xc01169b0a8, 0x519bbc, 0xc055b5faa0, 0xc05b573860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc025dc81e0, {0x63c85b8, 0xc0430ca8a0}, 0xc064a2e900, 0xc01169b1a8, 0x55b5faa0, 0x0, 0x2, 0xc055b5faa0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0430ca8a0}, 0xc055b5f9e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04a866d80, 0xaa}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0430ca8a0}, 0x6352260, 0x50b35658) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0430ca8a0}, {0xc055b5f8c0, {0x17430110356c23ac, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0430ca8a0}, 0xc04e9179a0, 0xc01169b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0430ca8a0}, 0x660089, {0xc055b5f8c0, {0x17430110356c23ac, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0430ca8a0}, 0xc05ae7e800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110356c23ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110356c23ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0430ca7e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0430ca7e0}, {0xc01169d3f8, 0x660089}, 0xc01169d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0430ca7e0}, {0xc0015221e0}, 0xc05ae7e600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0430ca720}, 0xc05ae7e600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}, 0xc025c55580, 0xc047a252c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}, 0x3, 0xc01a080c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01169d980}}}, {0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}, 0xc025c55580, 0xc01a080c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0430ca720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0430ca720}, {0x40, 0xc01169db70}, 0xc01169dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0430ca720}, {0x4e9d7a0, 0xc05ae7e600}, 0xc025c55580, 0xc047a252c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0430ca720}, 0xc01aec2ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc055b5f7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc055b5f7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809184 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc041fb6570}, 0xc074a63ad0, 0x1, 0x70801803e5648b, 0x37e11d600, 0xc07b860cc0, 0xc0603d9800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041fb6570}, 0x120, 0xc0168bb230, 0x7b860c60, 0x0, 0x660069, 0xc07b860cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc041fb6570}, 0xc07b860c60, {{{{0x0, 0x0}, {0xc0262b7440, 0xe8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041fb6570}, 0x6352410, 0x1b3088b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc041fb6570}, {0xc07b860b40, {0x1743011066fad2c7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc041fb6570}, 0xc083df4320, 0xc0168bb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041fb6570}, 0x660089, {0xc07b860b40, {0x1743011066fad2c7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc041fb6570}, 0xc0824c4c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066fad2c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066fad2c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041fb64b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041fb64b0}, {0xc0168bd3f8, 0x660089}, 0xc0168bd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041fb64b0}, {0xc0015221e0}, 0xc0824c4a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041fb6390}, 0xc0824c4a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}, 0xc06db6f060, 0xc074a63aa0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}, 0x3, 0xc06719fcc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0168bd980}}}, {0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}, 0xc06db6f060, 0xc06719fcc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041fb6390}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041fb6390}, {0x40, 0xc0168bdb70}, 0xc0168bdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041fb6390}, {0x4e9d7a0, 0xc0824c4a00}, 0xc06db6f060, 0xc074a63aa0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041fb6390}, 0xc07a062540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07b860a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07b860a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806150 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc03c07c120}, 0xc08aad8f00, 0x1, 0x708017e90411aa, 0x37e11d600, 0xc0912fa060, 0xc059013140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03c07c120}, 0x1e0, 0xc006a6b230, 0x912fa000, 0x0, 0x660069, 0xc0912fa060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc03c07c120}, 0xc0912fa000, {{{{0x0, 0x0}, {0xc08a57dd40, 0xfc}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc053192de0, {0x63c85b8, 0xc03c07c120}, 0xc0244a77f0, 0x6a6b428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc03c07c120}, {0xc0784b7b00, {0x174301104bfd6c98, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc03c07c120}, 0xc03289a6e0, 0xc006a6b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03c07c120}, 0x660089, {0xc0784b7b00, {0x174301104bfd6c98, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc03c07c120}, 0xc06749ee00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104bfd6c98, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104bfd6c98, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c07c090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c07c090}, {0xc006a6d3f8, 0x660089}, 0xc006a6d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c07c090}, {0xd0300a280a200118}, 0xc06749ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c07c000}, 0xc06749ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}, 0xc076b100c0, 0xc08aad8ed0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}, 0x3, 0xc027088700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006a6d980}}}, {0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}, 0xc076b100c0, 0xc027088700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c07c000}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c07c000}, {0x40, 0xc006a6db70}, 0xc006a6dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c07c000}, {0x4e9d7a0, 0xc06749ec00}, 0xc076b100c0, 0xc08aad8ed0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c07c000}, 0xc01d9423c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc035f0d320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc035f0d320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810389 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc05c34ce10}, 0xc024fc0378, 0x1, 0xc0190490a8, 0x519bbc, 0xc057ea0d80, 0xc067489440) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc071702330, {0x63c85b8, 0xc05c34ce10}, 0xc08f30f200, 0xc0190491a8, 0x21, 0x30, 0x0, 0xc057ea0d80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc05c34ce10}, 0xc057ea0d20, {{{{0xc044876480, 0x10}, {0xc08fc2ab40, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x30, {0x63c85b8, 0xc05c34ce10}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc05c34ce10}, {0xc05a840d80, {0x174301106c169258, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc05c34ce10}, 0xc055fad2c0, 0xc0190499f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05c34ce10}, 0x660089, {0xc05a840d80, {0x174301106c169258, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc05c34ce10}, 0xc08ad6fa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c169258, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c169258, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05c34cd80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c34cd80}, {0xc01904b3f8, 0x660089}, 0xc01904b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05c34cd80}, {0x0}, 0xc08ad6f800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05c34ccf0}, 0xc08ad6f800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}, 0xc07252b0e0, 0xc03d273350) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}, 0x3, 0xc073febf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01904b980}}}, {0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}, 0xc07252b0e0, 0xc073febf00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05c34ccf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c34ccf0}, {0x40, 0xc01904bb70}, 0xc01904bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05c34ccf0}, {0x4e9d7a0, 0xc08ad6f800}, 0xc07252b0e0, 0xc03d273350) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05c34ccf0}, 0xc0b59fed20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05a840b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05a840b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219282951 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0413d2a50, 0x12c) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc073314000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc073314660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06b992000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06b992000, {0x63c8510, 0xc0413d2a80}, 0xc058f23dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07a89fb00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0413d2a80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07a89fb00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808546 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc07bb08600}, 0xc0274bd218, 0x1, 0xc00663d0a8, 0x519bbc, 0xc04ab38600, 0xc038478600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07bb08600}, 0x1e0, 0xc00663d230, 0x4ab385a0, 0x0, 0x660069, 0xc04ab38600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc07bb08600}, 0xc04ab385a0, {{{{0x0, 0x0}, {0xc044af2b40, 0x50}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07bb08600}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc07bb08600}, {0xc05fd44000, {0x174301105d4dee7b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc07bb08600}, 0xc07471b540, 0xc00663d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07bb08600}, 0x660089, {0xc05fd44000, {0x174301105d4dee7b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07bb08600}, 0xc086ec9c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105d4dee7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105d4dee7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07bb08570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bb08570}, {0xc00663f3f8, 0x660089}, 0xc00663f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07bb08570}, {0x7fe3c14f94d8}, 0xc086ec9a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07bb084b0}, 0xc086ec9a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}, 0xc062408960, 0xc01f700870) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}, 0x3, 0xc0750a6c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00663f980}}}, {0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}, 0xc062408960, 0xc0750a6c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07bb084b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bb084b0}, {0x40, 0xc00663fb70}, 0xc00663fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07bb084b0}, {0x4e9d7a0, 0xc086ec9a00}, 0xc062408960, 0xc01f700870) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07bb084b0}, 0xc034ed6ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05eb19e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05eb19e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811694 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc03b7b2510}, 0xc02902ccd8, 0x1, 0xc03b89f6d8, 0x519bbc, 0xc036053f20, 0xc060e6e540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01c8e9f20, {0x63c85b8, 0xc03b7b2510}, 0xc0477b5c20, 0xc03b89f7d8, 0x36053f20, 0x0, 0x0, 0xc036053f20, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc03b7b2510}, 0xc036053e60, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc042a73200, 0xb0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03b7b2510}, 0x6352260, 0x36ce8188) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc03b7b2510}, {0xc036053d40, {0x174301105b1ef448, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc03b7b2510}, 0xc07390d860, 0xc03b8a0020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03b7b2510}, 0x660089, {0xc036053d40, {0x174301105b1ef448, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03b7b2510}, 0xc04fb2d000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b1ef448, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b1ef448, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03b7b2480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b7b2480}, {0xc06318ba28, 0x660089}, 0xc06318ba30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03b7b2480}, {0xc00102e4e0}, 0xc04fb2cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03b7b2420}, 0xc04fb2cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0824bd8d0, {0x63c85b8, 0xc03b7b2360}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0824bd8d0, {0x63c85b8, 0xc03b7b2360}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc03b7b2360}, {0x4f67ad2, 0x10beec0}, 0xc01f3f0fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219789351 [runnable]: internal/poll.runtime_pollWait(0x7fe422a59a90, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc070b11280, 0xc04f81a800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc070b11280, {0xc04f81a800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc070b11280, {0xc04f81a800, 0x735, 0xc049a22280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0603b9848, {0xc04f81a800, 0x7eda79, 0xc04f81a805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc032c83440, {0xc04f81a800, 0x35318b4, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc05bec8fa8, {0xc04f81a800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc058353778, {0x631b900, 0xc05bec8fa8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc058353500, {0x7fe42ab59310, 0xc032c83440}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc058353500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc058353500, {0xc08eb73000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01e72f7e8, {0xc08eb73000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc078274b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc078274b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0782750e0, {0x634aac0, 0xc078274b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc078274a80, 0xc0199af930, 0xc035adf42b, 0xc035adf42d, {0x635bec0, 0xc0c3d1a7e0}, 0xc000d6ae00, 0xc0183f0680, 0xc035adf429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc078274a80, {0x63c85b8, 0xc03bbdf680}, 0xc0199af8f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06e8e1047, 0x8}}, 0x0, 0xc03bbdf620, 0xc03bbdf650, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03bbdf680}, {0x643d810, 0xc032c83440}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0314dc870}, {0x643d810, 0xc032c83440}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0314dc870}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219150600 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc000cb3540, {0x643dc30, 0xc014670110}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:402 +0x20e github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x4ddf7a0, 0xc000cb3540}, {0x6400fe0, 0xc0b844b540}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:170 +0x9f github.com/cockroachdb/cockroach/pkg/util/tracing.StreamServerInterceptor.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}, 0xc02acf3998, 0x51c4398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:204 +0x6c2 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1408 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func4({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}, 0x3, 0xc049a57600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:247 +0x74 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({{{0x16f0987}}}, {0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}, 0xc02acf3998, 0xc049a57600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:98 +0x1c3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2.1({0x0, 0x456914}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:216 +0x2d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050e34f90}, {0x460ef4, 0xc01f835bc8}, 0xc01f835b70) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func2({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}, 0xc02acf3998, 0xc049a57600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:215 +0xf3 google.golang.org/grpc.chainStreamInterceptors.func1.1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1411 +0x83 google.golang.org/grpc.chainStreamInterceptors.func1({0x4ddf7a0, 0xc000cb3540}, {0x6401850, 0xc04ee3ef70}, 0xc02acf3998, 0x51c4398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1413 +0x12b google.golang.org/grpc.(*Server).processStreamingRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0597b4ea0, 0xc007b7af30, 0x9191c80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1549 +0xe8e google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0597b4ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1624 +0x9e5 google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803869 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc07d081b90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800370 [runnable]: sync.runtime_SemacquireMutex(0xc092916690, 0x8f, 0x9bb12c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc092916538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc092916560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0585d1b00, {0xffffffffffffff15, 0xc0929166c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0585d1b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0585d1b00, {0xc025590780, 0x32, 0x74}, {0xc0225d7400, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07432a898, {0xc025590780, 0x32, 0x74}, {0xc0225d7400, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0743291c0, {0xc0225d7400, 0x72, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0743291c0, {0xc025590780, 0x72, 0xc0225d7400}, {0xc0225d7400, 0xc02ca72d80, 0x31}, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc074328f00, {0xc0225d7400, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0459fd428, {{0xc02ca72d80, 0x31, 0x80}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07afa9700, {{0xc0280be2d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0562b0000, {0x63c85b8, 0xc04103a990}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04103a990}, {0x64a6560, 0xc07afa9700}, {0xc0280be2d0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04103a990}, {0x6466ed8, 0xc0459fd180}, {0xc0280be2d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c77e9c0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc020683300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ab8000, {0x63c85b8, 0xc04103a990}, 0xc020683300, 0xc0463c1180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ab8000, {0x63c85b8, 0xc04103a990}, 0xc020683300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04103a900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04103a900}, {0xc0076c88f0, 0x660089}, 0xc0076c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04103a900}, {0xc00102e4e0}, 0xc020683200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04103a8a0}, 0xc020683200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b3b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806983 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc03af96540}, 0xc083a1cfc0, 0x1, 0xc003f770a8, 0x519bbc, 0xc015041c80, 0xc06f4ded80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc03af96540}, 0x1e0, 0xc003f77230, 0x15041c20, 0x0, 0x660069, 0xc015041c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc03af96540}, 0xc015041c20, {{{{0x0, 0x0}, {0xc082ccd200, 0x60}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc03af96540}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc03af96540}, {0xc0718bd560, {0x174301104eeebc3b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc03af96540}, 0xc08e180dc0, 0xc003f779f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03af96540}, 0x660089, {0xc0718bd560, {0x174301104eeebc3b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc03af96540}, 0xc07d41e000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104eeebc3b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104eeebc3b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03af964b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03af964b0}, {0xc003f793f8, 0x660089}, 0xc003f79400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03af964b0}, {0x40df00bf96283369}, 0xc078189e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03af96420}, 0xc078189e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}, 0xc090c20ea0, 0xc020230300) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}, 0x3, 0xc090051800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003f79980}}}, {0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}, 0xc090c20ea0, 0xc090051800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03af96420}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03af96420}, {0x40, 0xc003f79b70}, 0xc003f79ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03af96420}, {0x4e9d7a0, 0xc078189e00}, 0xc090c20ea0, 0xc020230300) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03af96420}, 0xc04c216720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0718bd440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0718bd440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801179 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01f18cc00, {0x63c85b8, 0xc078273050}, 0xc07d4b3d00, 0xc02ac345a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f18cc00, {0x63c85b8, 0xc078273050}, 0xc07d4b3d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110196f8284, 0x0, 0x0}, 0xc043ec6ac0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110196f8284, 0x0, 0x0}, 0xc043ec6ac0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078272fc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078272fc0}, {0xc095ad5918, 0x660089}, 0xc095ad5920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078272fc0}, {0xc00102e4e0}, 0xc07d4b3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078272f60}, 0xc07d4b3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc03c5e3020}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc078272ed0}, {0xc00dfd36b0, 0x24}, 0x48bf84f2e, 0xc002ca3d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x4c1af3}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc078272ed0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801933 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc04ef9e810}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 214515061 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe99698, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc068681200, 0xc00ff69000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc068681200, {0xc00ff69000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc068681200, {0xc00ff69000, 0x735, 0xc0436bd2c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc020d69c38, {0xc00ff69000, 0x7eda79, 0xc00ff69005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06b889b60, {0xc00ff69000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a90828, {0xc00ff69000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04543b778, {0x631b900, 0xc043a90828}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04543b500, {0x7fe42ab59310, 0xc06b889b60}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04543b500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04543b500, {0xc04a386000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06cad3020, {0xc04a386000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0236d9588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0236d9588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0236d9b60, {0x634aac0, 0xc0236d9588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0236d9500, 0xc04207d160, 0xc09444342b, 0xc09444342d, {0x635bec0, 0xc0610e19e0}, 0xc000d6ae00, 0xc03ee312a8, 0xc094443429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0236d9500, {0x63c85b8, 0xc0381c1050}, 0xc04207d150, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06b5a603c, 0x8}}, 0x0, 0xc0381c0ff0, 0xc0381c1020, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0381c1050}, {0x643d810, 0xc06b889b60}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02eb4ec90}, {0x643d810, 0xc06b889b60}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02eb4ec90}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812515 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790590 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc05f452c00}, 0xc015270cb0, {0x646cc48, 0xc05f452c60}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc05f452c00}, 0xedb7a834e, 0xc046065320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc05f452c00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06e426a50}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc015270ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06e426a50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219776627 [IO wait]: internal/poll.runtime_pollWait(0x7fdb124bc4f0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc079665080, 0xc0715e3980, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc079665080, {0xc0715e3980, 0x1854, 0x1854}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc079665080, {0xc0715e3980, 0x1854, 0xc0b5b2c500}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc025ead820, {0xc0715e3980, 0x7eda79, 0xc0715e3985}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02ec21e00, {0xc0715e3980, 0xc042698c60, 0x1854}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0bb3a9c68, {0xc0715e3980, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc058b653f8, {0x631b900, 0xc0bb3a9c68}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc058b65180, {0x7fe42ab59310, 0xc02ec21e00}, 0x1854) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc058b65180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc058b65180, {0xc0a67a3000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0210f7278, {0xc0a67a3000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06ca14088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06ca14088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06ca14660, {0x634aac0, 0xc06ca14088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06ca14000, 0xc042ac1e30, 0xc04269942b, 0xc04269942d, {0x635bec0, 0xc073ebeb40}, 0xc000d6ae00, 0xc03f49dee4, 0xc042699429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06ca14000, {0x63c85b8, 0xc06c97f110}, 0xc042ac1e20, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc013ff2009, 0x8}}, 0x0, 0xc06c97f0b0, 0xc06c97f0e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06c97f110}, {0x643d810, 0xc02ec21e00}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07190cb40}, {0x643d810, 0xc02ec21e00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07190cb40}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812248 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0201e3900, {0x63c85b8, 0xc059b687e0}, 0xc0a97ede60, 0x1, 0xc00564d0a8, 0x519bbc, 0xc03fdb0ba0, 0xc05063e180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc059b687e0}, 0x120, 0xc00564d230, 0x3fdb0b40, 0x0, 0x660069, 0xc03fdb0ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0201e3900, {0x63c85b8, 0xc059b687e0}, 0xc03fdb0b40, {{{{0x0, 0x0}, {0xc0425e7560, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc059b687e0}, 0x6352410, 0x434f4740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01f8c5810, {0x63c85b8, 0xc059b687e0}, {0xc03fdb0a20, {0x174301107f0ff4ae, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01f8c5810, {0x63c85b8, 0xc059b687e0}, 0xc075833360, 0xc00564d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc059b687e0}, 0x660089, {0xc03fdb0a20, {0x174301107f0ff4ae, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020433c00, {0x63c85b8, 0xc059b687e0}, 0xc079a54f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f0ff4ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f0ff4ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059b68750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059b68750}, {0xc00564f3f8, 0x660089}, 0xc00564f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059b68750}, {0x7fe426abc560}, 0xc079a54d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc059b68660}, 0xc079a54d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}, 0xc0413b7440, 0xc030c9ccc0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}, 0x3, 0xc07b173400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00564f980}}}, {0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}, 0xc0413b7440, 0xc07b173400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc059b68660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059b68660}, {0x40, 0xc00564fb70}, 0xc00564fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc059b68660}, {0x4e9d7a0, 0xc079a54d00}, 0xc0413b7440, 0xc030c9ccc0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc059b68660}, 0xc02f9c6540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03fdb0900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03fdb0900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802543 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e099c00, {0x63c85b8, 0xc043585f80}, 0xc04c765600, 0xc022bbb540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099c00, {0x63c85b8, 0xc043585f80}, 0xc04c765600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffcaca4d5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffcaca4d5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043585dd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043585dd0}, {0xc0581619c0, 0x660089}, 0xc0581619c8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043585dd0}, {0xc00102e4e0}, 0xc04c765500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043585d70}, 0xc04c765500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x63c85b8, 0xc043585b00}, {0x7fe422a51230, 0xc070cec600}, 0xc01d58bb00, {0xc043be2000, 0x3, 0x4}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x24d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x63c85b8, 0xc043585b00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219789413 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc04d8af020}, 0xc01a1b9cb0, {0x646cc48, 0xc04d8af1a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc04d8af020}, 0xedb7a834d, 0xc092baa360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc04d8af020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03c1041e0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01a1b9ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc03c1041e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805131 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc042d988d0}, 0xc025094798, 0x1, 0x0, 0x1, 0xc07279d260, 0xc04abd9020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc042d988d0}, 0x120, 0xc07c256728, 0x7279d200, 0xc0375b0ea0, 0x18, 0xc07279d260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc042d988d0}, 0xc07279d200, {{{{0x0, 0x0}, {0xc0477b4ea0, 0x19}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc042d988d0}, 0x6352410, 0x71c3e9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc042d988d0}, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc042d988d0}, 0xc0734c0500, 0xc075e6eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc042d988d0}, 0x660089, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc042d988d0}, 0xc055bf7200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042d98840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042d98840}, {0xc0032fa8f0, 0x660089}, 0xc0032fa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042d98840}, {0xc00102e4e0}, 0xc055bf7100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042d987e0}, 0xc055bf7100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02a728cc0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803730 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc06125cd50}, 0xc04f9e9e78, 0x1, 0xc0769b30a8, 0x519bbc, 0xc042e863c0, 0xc03fb6e2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06125cd50}, 0x120, 0xc0769b3230, 0x42e86360, 0x0, 0x660069, 0xc042e863c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc06125cd50}, 0xc042e86360, {{{{0x0, 0x0}, {0xc090f2a900, 0x19}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06125cd50}, 0x6352410, 0x473dd648) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc06125cd50}, {0xc042e86240, {0x174301102e330700, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc06125cd50}, 0xc04e1d60a0, 0xc0769b39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06125cd50}, 0x660089, {0xc042e86240, {0x174301102e330700, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc06125cd50}, 0xc040e40700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e330700, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e330700, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06125cc90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06125cc90}, {0xc0769b53f8, 0x660089}, 0xc0769b5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06125cc90}, {0x0}, 0xc040e40500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06125cb70}, 0xc040e40500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}, 0xc01ba24b40, 0xc0319db4d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}, 0x3, 0xc07ae77b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0769b5980}}}, {0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}, 0xc01ba24b40, 0xc07ae77b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06125cb70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06125cb70}, {0x40, 0xc0769b5b70}, 0xc0769b5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06125cb70}, {0x4e9d7a0, 0xc040e40500}, 0xc01ba24b40, 0xc0319db4d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06125cb70}, 0xc0299b39e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041e02360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc041e02360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219149964 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream.func2() github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:385 +0x77 created by github.com/cockroachdb/cockroach/pkg/util/tracing.newTracingClientStream github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:384 +0x15f goroutine 219804415 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc071d2cb70}, 0xc04e4406d8, 0x1, 0xc074f5f0a8, 0x519bbc, 0xc0504a0060, 0xc04c117e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc071d2cb70}, 0x1e0, 0xc074f5f230, 0x504a0000, 0x0, 0x660069, 0xc0504a0060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc071d2cb70}, 0xc0504a0000, {{{{0x0, 0x0}, {0xc01fe83440, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc071d2cb70}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc071d2cb70}, {0xc05c325e60, {0x17430110315778ce, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc071d2cb70}, 0xc0667475e0, 0xc074f5f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc071d2cb70}, 0x660089, {0xc05c325e60, {0x17430110315778ce, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc071d2cb70}, 0xc0503df100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110315778ce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110315778ce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc071d2cae0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071d2cae0}, {0xc074f613f8, 0x660089}, 0xc074f61400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc071d2cae0}, {0xc0524745a0}, 0xc0503def00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc071d2ca50}, 0xc0503def00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}, 0xc05393f6c0, 0xc064a06570) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}, 0x3, 0xc0cc383ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc074f61980}}}, {0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}, 0xc05393f6c0, 0xc0cc383ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc071d2ca50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071d2ca50}, {0x40, 0xc074f61b70}, 0xc074f61ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc071d2ca50}, {0x4e9d7a0, 0xc0503def00}, 0xc05393f6c0, 0xc064a06570) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc071d2ca50}, 0xc07374e480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05c325d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05c325d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218824287 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4e0d8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc02e7e3980, 0xc0a226c000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02e7e3980, {0xc0a226c000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc02e7e3980, {0xc0a226c000, 0x735, 0xc04ab2adc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc065f8be88, {0xc0a226c000, 0x7eda79, 0xc0a226c005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02c77bda0, {0xc0a226c000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0558d00a8, {0xc0a226c000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc058609af8, {0x631b900, 0xc0558d00a8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc058609880, {0x7fe42ab59310, 0xc02c77bda0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc058609880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc058609880, {0xc0b7596000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc048ac05d0, {0xc0b7596000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06f8e2088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06f8e2088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06f8e2660, {0x634aac0, 0xc06f8e2088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06f8e2000, 0xc040550f60, 0xc07d29542b, 0xc07d29542d, {0x635bec0, 0xc039c91830}, 0xc000d6ae00, 0xc036474ce8, 0xc07d295429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06f8e2000, {0x63c85b8, 0xc075ec2150}, 0xc040550f50, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0b7632038, 0x8}}, 0x0, 0xc075ec20f0, 0xc075ec2120, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc075ec2150}, {0x643d810, 0xc02c77bda0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07adde570}, {0x643d810, 0xc02c77bda0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07adde570}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219151369 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc000cb3540, 0x63c8510, 0xc0886de7e0, 0xc0181fa800, {0x643dbd8, 0xc02c02bb50}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:507 +0x396 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x63c85b8, 0xc049b61110}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:673 +0x385 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812765 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812678 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc082c85500, 0xc028aa9c28) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc082c85500, {0xc02c195e80, 0x1e, 0x77}, {0xc00408fb90, 0x1e, 0x6f}, 0x0, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc088aa7f88, {0xc02c195e80, 0x1, 0x6f}, {0xc00408fb90, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc088aa7f88, {0xc02c195e80, 0x1e, 0x77}, {0xc00408fb90, 0x1e, 0x6f}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc088aa76c0, {0xc00408fb90, 0xc028aaa9e0, 0x11a3c29}, 0x7fe429c0e8c0, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc088aa76c0, {0xc02c195e80, 0x1e, 0x6f}, {0xc00408fb90, 0x1d, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc088aa7400, {0xc00408fb90, 0x1e, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc060611708, {{0xc068a3f7c0, 0x1d, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc041b7e000, {0x63c85b8, 0xc050520c90}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc050520c90}, {0x64a6618, 0xc060611708}, {0xc068a3f7c0, 0x0, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc050520c90}, {0x6466ed8, 0xc060611700}, {0xc068a3f7c0, 0x1d, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x63c85b8, 0xc050520c90}, {0x6466ed8, 0xc060611700}, {0xc068a3f7c0, 0xc028aaaac8, 0x461247}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:617 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0, {0x63c85b8, 0xc050520c90}, {0x6466ed8, 0xc060611700}, {0xec, 0x20, 0x10, 0xb0, 0xd2, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:98 +0xdb github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0xec, 0x20, 0x10, 0xb0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:1958 +0xbc github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0711f0700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:201 +0x37e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a0bc8de6, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc016654000, {0x63c85b8, 0xc050520c90}, 0xc0711f0700, 0xc053b56960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016654000, {0x63c85b8, 0xc050520c90}, 0xc0711f0700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011082ef28e6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011082ef28e6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050520c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050520c00}, {0xc028aaf3f8, 0x660089}, 0xc028aaf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050520c00}, {0x0}, 0xc0711f0500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050520b70}, 0xc0711f0500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}, 0xc00abd3100, 0xc017daeea0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}, 0x3, 0xc06b3d6bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028aaf980}}}, {0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}, 0xc00abd3100, 0xc06b3d6bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc050520b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050520b70}, {0x40, 0xc028aafb70}, 0xc028aafab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc050520b70}, {0x4e9d7a0, 0xc0711f0500}, 0xc00abd3100, 0xc017daeea0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc050520b70}, 0xc02e5a0060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0584bb8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0584bb8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803757 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc03ff5cff0}, 0xc0651b4300, 0xc00d503c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03ff5cff0}, 0xc0651b4300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fcc5c7a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fcc5c7a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03ff5cf60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ff5cf60}, {0xc075a273f8, 0x660089}, 0xc075a27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03ff5cf60}, {0x0}, 0xc05977df00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03ff5cea0}, 0xc05977df00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}, 0xc03d18bd40, 0xc0124608a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}, 0x3, 0xc056f03640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075a27980}}}, {0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}, 0xc03d18bd40, 0xc056f03640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03ff5cea0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ff5cea0}, {0x40, 0xc075a27b70}, 0xc075a27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03ff5cea0}, {0x4e9d7a0, 0xc05977df00}, 0xc03d18bd40, 0xc0124608a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03ff5cea0}, 0xc06b65ac00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0460e1440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0460e1440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808191 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc02fc9a8a0}, 0xc02b1cdc20, 0x1, 0xc079ef5f20, 0x519bbc, 0xc03e6a0360, 0xc06f706d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0, {0x63c85b8, 0xc02fc9a8a0}, 0xc0, 0xc079ef60a8, 0x3e6a0300, 0x0, 0x660069, 0xc03e6a0360, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc02fc9a8a0}, 0xc03e6a0300, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc02fc9a8a0}, 0x63524a0, 0x603b8118) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc02fc9a8a0}, {0x0, {0x174301105cb41a0e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02fc9a8a0}, 0xc07839c140, 0x1) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02fc9a8a0}, 0x660089, {0x0, {0x174301105cb41a0e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02fc9a8a0}, 0xc0625e8400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02fc9a810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fc9a810}, {0xc005d6c270, 0x660089}, 0xc005d6c278) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02fc9a810}, {0xc02fc9a480}, 0xc0625e8300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02fc9a7b0}, 0xc0625e8300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x174301105cb41a0e, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc02fc9a630}, 0x0, 0xc081f95180) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc02fc9a630}, 0x66040f) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).queryTxnStatus(0xc019919860, {0x63c85b8, 0xc02fc9a630}, {{0x5c, 0x4, 0x96, 0xea, 0xaf, 0xab, 0x4b, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:922 +0x1b0 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).waitForPush(0xc019919860, {0x63c85b8, 0xc02fc9a630}, 0xc04ec64d20, 0xc0156d1e80, 0xc03d04ca00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:610 +0xd4a github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush.func2({0x63c85b8, 0xc02fc9a630}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:509 +0x52 runtime/pprof.Do({0x63c85b8, 0xc02fc9a5a0}, {{0xc01e9a8f00, 0x24, 0x24}}, 0xc005d6f380) GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc019919860, {0x63c85b8, 0xc02fc9a5a0}, 0xc04ec64d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:508 +0xb6c github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0301af310, {0x63c85b8, 0xc02fc9a5a0}, {0x0, {0x174301105cadefe5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:347 +0xa9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02fc9a5a0}, 0xc085824140, 0xc005d6f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:250 +0x1fc github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02fc9a5a0}, 0x660089, {0x0, {0x174301105cadefe5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02fc9a5a0}, 0xc0625e8200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105cadefe5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105cadefe5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301105cadefe5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02fc9a510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fc9a510}, {0xc005d713f8, 0x660089}, 0xc005d71400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02fc9a510}, {0x7fe3ac1da4d8}, 0xc0625e8000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02fc9a480}, 0xc0625e8000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}, 0xc0156d1e40, 0xc015393cb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}, 0x3, 0xc01e9a8780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005d71980}}}, {0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}, 0xc0156d1e40, 0xc01e9a8780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02fc9a480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02fc9a480}, {0x40, 0xc005d71b70}, 0xc005d71ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02fc9a480}, {0x4e9d7a0, 0xc0625e8000}, 0xc0156d1e40, 0xc015393cb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02fc9a480}, 0xc020c40ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc074d8c120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc074d8c120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800400 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0c74cf740}, 0xc02866c700, 0xc04646b680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0c74cf740}, 0xc02866c700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff7584398, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff7584398, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c74cf6b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c74cf6b0}, {0xc042cdf3f8, 0x660089}, 0xc042cdf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c74cf6b0}, {0x7fe429307910}, 0xc071b6fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c74cf620}, 0xc071b6fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}, 0xc0661a7c40, 0xc06c9b5b00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}, 0x3, 0xc00748c340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042cdf980}}}, {0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}, 0xc0661a7c40, 0xc00748c340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0c74cf620}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c74cf620}, {0x40, 0xc042cdfb70}, 0xc042cdfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0c74cf620}, {0x4e9d7a0, 0xc071b6fe00}, 0xc0661a7c40, 0xc06c9b5b00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0c74cf620}, 0xc043fe6900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01b65a360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01b65a360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811908 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc01c2f1b00}, 0xc0d04a0e10, 0x1, 0xc03372d0a8, 0x519bbc, 0xc04094c2a0, 0xc0139e2de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01c2f1b00}, 0x120, 0xc03372d230, 0x4094c240, 0x0, 0x660069, 0xc04094c2a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc01c2f1b00}, 0xc04094c240, {{{{0x0, 0x0}, {0xc0aed58240, 0x32}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01c2f1b00}, 0x6352410, 0x3c90c0c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc01c2f1b00}, {0xc04094c120, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc01c2f1b00}, 0xc07c610aa0, 0xc03372d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01c2f1b00}, 0x660089, {0xc04094c120, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc01c2f1b00}, 0xc0aa691100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c2f1a70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c2f1a70}, {0xc03372f3f8, 0x660089}, 0xc03372f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c2f1a70}, {0x3feffff7b46c0157}, 0xc0aa690e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c2f19e0}, 0xc0aa690e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}, 0xc0bb7ea820, 0xc0d3545110) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}, 0x3, 0xc03ee79900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03372f980}}}, {0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}, 0xc0bb7ea820, 0xc03ee79900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01c2f19e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c2f19e0}, {0x40, 0xc03372fb70}, 0xc03372fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01c2f19e0}, {0x4e9d7a0, 0xc0aa690e00}, 0xc0bb7ea820, 0xc0d3545110) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01c2f19e0}, 0xc057f07ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04094c000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04094c000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807428 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc043701020}, 0xc0913cce00, 0xc01dfa5f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc043701020}, 0xc0913cce00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105741e855, 0x0, 0x0}, 0xc08c88f1f0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105741e855, 0x0, 0x0}, 0xc08c88f1f0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043700f90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043700f90}, {0xc0056d73f8, 0x660089}, 0xc0056d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043700f90}, {0x4052bff9d975114d}, 0xc0913ccc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043700f00}, 0xc0913ccc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}, 0xc072dd7a40, 0xc057bc63a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}, 0x3, 0xc04094aac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0056d7980}}}, {0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}, 0xc072dd7a40, 0xc04094aac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc043700f00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043700f00}, {0x40, 0xc0056d7b70}, 0xc0056d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc043700f00}, {0x4e9d7a0, 0xc0913ccc00}, 0xc072dd7a40, 0xc057bc63a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc043700f00}, 0xc035b84180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04ff57e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04ff57e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811922 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc024a55800}, 0xc0c5889158, 0x1, 0xc032c890a8, 0x519bbc, 0xc02af07ec0, 0xc0828fa720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc024a55800}, 0x120, 0xc032c89230, 0x2af07e60, 0x0, 0x660069, 0xc02af07ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc024a55800}, 0xc02af07e60, {{{{0x0, 0x0}, {0xc06210b200, 0x2b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc024a55800}, 0x6352410, 0x55c36070) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc024a55800}, {0xc02af07c20, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc024a55800}, 0xc084c761e0, 0xc032c899f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024a55800}, 0x660089, {0xc02af07c20, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc024a55800}, 0xc010bc4300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024a55740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024a55740}, {0xc032c8b3f8, 0x660089}, 0xc032c8b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024a55740}, {0x33}, 0xc010bc4100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024a55650}, 0xc010bc4100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}, 0xc07af6e8e0, 0xc04479ae58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}, 0x3, 0xc03d3a9f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032c8b980}}}, {0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}, 0xc07af6e8e0, 0xc03d3a9f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc024a55650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024a55650}, {0x40, 0xc032c8bb70}, 0xc032c8bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc024a55650}, {0x4e9d7a0, 0xc010bc4100}, 0xc07af6e8e0, 0xc04479ae58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc024a55650}, 0xc07756ca80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc02af079e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc02af079e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811121 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016485540, {0x63c85b8, 0xc0350e3500}, 0xc06427a510, 0x1, 0x70801810d4d08c, 0x37e11d600, 0xc00c4bf020, 0xc0255042a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0350e3500}, 0x120, 0xc028d4d230, 0xc4befc0, 0x0, 0x660069, 0xc00c4bf020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016485540, {0x63c85b8, 0xc0350e3500}, 0xc00c4befc0, {{{{0x0, 0x0}, {0xc073da2d80, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0350e3500}, 0x6352410, 0x47a9c700) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0164dc500, {0x63c85b8, 0xc0350e3500}, {0xc00c4beb40, {0x1743011073f5458b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0164dc500, {0x63c85b8, 0xc0350e3500}, 0xc03b365c20, 0xc028d4d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0350e3500}, 0x660089, {0xc00c4beb40, {0x1743011073f5458b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016515000, {0x63c85b8, 0xc0350e3500}, 0xc06df83800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011073f5458b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011073f5458b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0350e3470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0350e3470}, {0xc028d4f3f8, 0x660089}, 0xc028d4f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0350e3470}, {0xc0015221e0}, 0xc06df83600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0350e33e0}, 0xc06df83600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}, 0xc00cd966c0, 0xc04916d9f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}, 0x3, 0xc042b0a400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028d4f980}}}, {0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}, 0xc00cd966c0, 0xc042b0a400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0350e33e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0350e33e0}, {0x40, 0xc028d4fb70}, 0xc028d4fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0350e33e0}, {0x4e9d7a0, 0xc06df83600}, 0xc00cd966c0, 0xc04916d9f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0350e33e0}, 0xc06026d740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00c4be6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00c4be6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802737 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016f5da40, {0x63c85b8, 0xc027d11e30}, 0xc063a51e18, 0x1, 0xc07375d0a8, 0x519bbc, 0xc03f2f8a80, 0xc038389c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc027d11e30}, 0x120, 0xc07375d230, 0x3f2f8a20, 0x0, 0x660069, 0xc03f2f8a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016f5da40, {0x63c85b8, 0xc027d11e30}, 0xc03f2f8a20, {{{{0x0, 0x0}, {0xc02c03c900, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc027d11e30}, 0x6352410, 0x33b74c98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016fda500, {0x63c85b8, 0xc027d11e30}, {0xc03f2f8900, {0x174301102df8528a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016fda500, {0x63c85b8, 0xc027d11e30}, 0xc047cd5c20, 0xc07375d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc027d11e30}, 0x660089, {0xc03f2f8900, {0x174301102df8528a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017050000, {0x63c85b8, 0xc027d11e30}, 0xc06e133100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102df8528a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102df8528a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027d11da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027d11da0}, {0xc07375f3f8, 0x660089}, 0xc07375f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027d11da0}, {0x0}, 0xc06e132f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027d11bf0}, 0xc06e132f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}, 0xc04f1af960, 0xc09e45ed68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}, 0x3, 0xc038d02580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07375f980}}}, {0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}, 0xc04f1af960, 0xc038d02580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc027d11bf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027d11bf0}, {0x40, 0xc07375fb70}, 0xc07375fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc027d11bf0}, {0x4e9d7a0, 0xc06e132f00}, 0xc04f1af960, 0xc09e45ed68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc027d11bf0}, 0xc05ec70d20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03f2f87e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03f2f87e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802539 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0553efc20}, 0xc003501cb0, {0x646cc48, 0xc0553efc80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0553efc20}, 0xedb7a834f, 0xc038e7d200) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0553efc20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0408a1c50}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc003501ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0408a1c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812406 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0306077c0, {0x63c85b8, 0xc075ea2600}, 0xc0204cd3c8, 0x1, 0xc03d2f7be0, 0x519bbc, 0xc0a290bc60, 0xc059aa2f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc059d00900, {0x63c85b8, 0xc075ea2600}, 0xc0a290bea0, 0xc03d2f7ce0, 0x31, 0x40, 0x1, 0xc0a290bc60, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0306077c0, {0x63c85b8, 0xc075ea2600}, 0xc0a290bc00, {{{{0x0, 0x0}, {0xc071cea480, 0x1}}, {{0xc0140bac60, 0x2}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc075ea2600}, 0x63521a0, 0x7431b988) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc03063c320, {0x63c85b8, 0xc075ea2600}, {0xc025846d80, {0x174301107aeaabed, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc03063c320, {0x63c85b8, 0xc075ea2600}, 0xc0b16d6000, 0xc03d2f8528) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc075ea2600}, 0x660089, {0xc025846d80, {0x174301107aeaabed, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030641c00, {0x63c85b8, 0xc075ea2600}, 0xc079620900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107aeaabed, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107aeaabed, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075ea2540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075ea2540}, {0xc03d2f9f30, 0x660089}, 0xc03d2f9f38) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075ea2540}, {0xc00102e4e0}, 0xc079620800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc075ea24e0}, 0xc079620800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:197 +0x6a5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).rollback.func1.1({0x63c8580, 0xc04d5f63c0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:910 +0x65 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc075ea21e0}, {0x4f101e9, 0x12}, 0xdf8475800, 0xc07e11adf8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).rollback.func1({0x63c85b8, 0xc075ea21e0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:908 +0x165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812174 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc04ee573e0}, 0xc056cc7848, 0x1, 0xc03549d0a8, 0x519bbc, 0xc02edc3b60, 0xc0da5a62a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04ee573e0}, 0x120, 0xc03549d230, 0x2edc3b00, 0x0, 0x660069, 0xc02edc3b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc04ee573e0}, 0xc02edc3b00, {{{{0x0, 0x0}, {0xc0457ba6c0, 0x33}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ee573e0}, 0x6352410, 0x6b066b50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc04ee573e0}, {0xc02edc38c0, {0x174301107a113c4d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc04ee573e0}, 0xc08f98a000, 0xc03549d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ee573e0}, 0x660089, {0xc02edc38c0, {0x174301107a113c4d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc04ee573e0}, 0xc059186000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a113c4d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a113c4d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ee57350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ee57350}, {0xc03549f3f8, 0x660089}, 0xc03549f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ee57350}, {0x64}, 0xc082dd7e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ee57260}, 0xc082dd7e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}, 0xc04c987f60, 0xc047c08ed0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}, 0x3, 0xc0399524c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03549f980}}}, {0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}, 0xc04c987f60, 0xc0399524c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ee57260}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ee57260}, {0x40, 0xc03549fb70}, 0xc03549fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ee57260}, {0x4e9d7a0, 0xc082dd7e00}, 0xc04c987f60, 0xc047c08ed0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ee57260}, 0xc04ca3a960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc02edc3680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc02edc3680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810594 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc071f7c330}, 0xc0450f4270, 0x1, 0xc012dcf6d8, 0x519bbc, 0xc0104633e0, 0xc055ee2300) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04466d5c0, {0x63c85b8, 0xc071f7c330}, 0xc0731327e0, 0xc012dcf7d8, 0x104633e0, 0x0, 0x0, 0xc0104633e0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc071f7c330}, 0xc010463320, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc01b89b200, 0xa5}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc071f7c330}, 0x6352260, 0x20fd31a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc071f7c330}, {0xc0104630e0, {0x1743011032129bc8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc071f7c330}, 0xc05ad73220, 0xc012dd0020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc071f7c330}, 0x660089, {0xc0104630e0, {0x1743011032129bc8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc071f7c330}, 0xc06506d300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032129bc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032129bc8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc071f7c2a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071f7c2a0}, {0xc07dd21a28, 0x660089}, 0xc07dd21a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc071f7c2a0}, {0xc00102e4e0}, 0xc06506d200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc071f7c240}, 0xc06506d200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc09abfe750, {0x63c85b8, 0xc071f7c1b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc09abfe750, {0x63c85b8, 0xc071f7c1b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc071f7c1b0}, {0x4f67ad2, 0x10beec0}, 0xc01e253fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219810330 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02c4a8c00, {0x63c85b8, 0xc073042c90}, 0xc06ecedb00, 0xc052ea6820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc073042c90}, 0xc06ecedb00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d05fa06, 0x0, 0x0}, 0xc03fc6f7d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d05fa06, 0x0, 0x0}, 0xc03fc6f7d0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073042c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073042c00}, {0xc02807b3f8, 0x660089}, 0xc02807b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073042c00}, {0x7fe429f531a0}, 0xc06eced900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073042b70}, 0xc06eced900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}, 0xc060cbcf60, 0xc028879470) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}, 0x3, 0xc07067f580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02807b980}}}, {0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}, 0xc060cbcf60, 0xc07067f580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073042b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073042b70}, {0x40, 0xc02807bb70}, 0xc02807bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073042b70}, {0x4e9d7a0, 0xc06eced900}, 0xc060cbcf60, 0xc028879470) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073042b70}, 0xc01504ede0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc083494fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc083494fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808959 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc051761dd0}, 0xc05abce900, 0x1, 0xc01aa730a8, 0x519bbc, 0xc055fc2ba0, 0xc06eb53320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc051761dd0}, 0x1e0, 0xc01aa73230, 0x55fc2b40, 0x0, 0x660069, 0xc055fc2ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc051761dd0}, 0xc055fc2b40, {{{{0x0, 0x0}, {0xc04fe16b40, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc051761dd0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc051761dd0}, {0xc07acbdd40, {0x17430110618a8100, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc051761dd0}, 0xc070d93540, 0xc01aa739f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc051761dd0}, 0x660089, {0xc07acbdd40, {0x17430110618a8100, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc051761dd0}, 0xc0592f8d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110618a8100, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110618a8100, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc051761c80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051761c80}, {0xc01aa753f8, 0x660089}, 0xc01aa75400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc051761c80}, {0x100e00001005}, 0xc0592f8900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc051761bf0}, 0xc0592f8900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}, 0xc0737272c0, 0xc02b787ed8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}, 0x3, 0xc0238f4f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aa75980}}}, {0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}, 0xc0737272c0, 0xc0238f4f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc051761bf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc051761bf0}, {0x40, 0xc01aa75b70}, 0xc01aa75ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc051761bf0}, {0x4e9d7a0, 0xc0592f8900}, 0xc0737272c0, 0xc02b787ed8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc051761bf0}, 0xc045b16ea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07acbdc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07acbdc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810143 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024fb7800, {0x63c85b8, 0xc044117470}, 0xc088313100, 0xc02eccd680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024fb7800, {0x63c85b8, 0xc044117470}, 0xc088313100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c32aac8, 0x0, 0x0}, 0xc04a75ed80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c32aac8, 0x0, 0x0}, 0xc04a75ed80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0441173e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0441173e0}, {0xc01aafb3f8, 0x660089}, 0xc01aafb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0441173e0}, {0xc0015221e0}, 0xc088312f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0441172f0}, 0xc088312f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}, 0xc07a4032a0, 0xc03eec3638) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}, 0x3, 0xc07808e500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aafb980}}}, {0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}, 0xc07a4032a0, 0xc07808e500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0441172f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0441172f0}, {0x40, 0xc01aafbb70}, 0xc01aafbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0441172f0}, {0x4e9d7a0, 0xc088312f00}, 0xc07a4032a0, 0xc03eec3638) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0441172f0}, 0xc074255860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc062b2c360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc062b2c360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802735 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016d6d2c0, {0x63c85b8, 0xc04a0c61b0}, 0xc06e702cf0, 0x1, 0xc05fbab0a8, 0x519bbc, 0xc03bd495c0, 0xc03e7e2cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04a0c61b0}, 0x120, 0xc05fbab230, 0x3bd49560, 0x0, 0x660069, 0xc03bd495c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016d6d2c0, {0x63c85b8, 0xc04a0c61b0}, 0xc03bd49560, {{{{0x0, 0x0}, {0xc079f18c60, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04a0c61b0}, 0x6352410, 0x45bf44a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce3180, {0x63c85b8, 0xc04a0c61b0}, {0xc03bd49440, {0x174301102df8528a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce3180, {0x63c85b8, 0xc04a0c61b0}, 0xc07c810280, 0xc05fbab9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04a0c61b0}, 0x660089, {0xc03bd49440, {0x174301102df8528a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016efac00, {0x63c85b8, 0xc04a0c61b0}, 0xc038ced100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102df8528a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102df8528a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04a0c6120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a0c6120}, {0xc05fbad3f8, 0x660089}, 0xc05fbad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04a0c6120}, {0x705564363945627a}, 0xc038cecf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04a0c6090}, 0xc038cecf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}, 0xc035050fc0, 0xc0300687c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}, 0x3, 0xc01f3a8980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05fbad980}}}, {0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}, 0xc035050fc0, 0xc01f3a8980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04a0c6090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a0c6090}, {0x40, 0xc05fbadb70}, 0xc05fbadab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04a0c6090}, {0x4e9d7a0, 0xc038cecf00}, 0xc035050fc0, 0xc0300687c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04a0c6090}, 0xc01dc2bec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03f2f85a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03f2f85a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812750 [runnable]: syscall.Syscall6(0x11, 0x4f84, 0x7fdc58afd020, 0x2f4a, 0x27483d5, 0x0, 0x0) GOROOT/src/syscall/asm_linux_amd64.s:43 +0x5 syscall.Pread(0x0, {0x7fdc58afd020, 0x0, 0xc043b46de0}, 0x0) GOROOT/src/syscall/zsyscall_linux_amd64.go:1212 +0x58 internal/poll.(*FD).Pread(0xc0a7c89d40, {0x7fdc58afd020, 0x4ba25b, 0x2f4a}, 0x200000003) GOROOT/src/internal/poll/fd_unix.go:193 +0xf9 os.(*File).pread(...) GOROOT/src/os/file_posix.go:41 os.(*File).ReadAt(0xc013b805c0, {0x7fdc58afd020, 0xc043b46ec8, 0x1184134}, 0x2f6a) GOROOT/src/os/file.go:137 +0x191 github.com/cockroachdb/pebble/vfs.enospcFile.ReadAt(...) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_full.go:361 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0177622c0, {0x27483d5, 0x2f45}, 0x0, 0xc00a9eab78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2339 +0x3f2 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc00a9ea900, {0xffffffffffffff15, 0xc043b470c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc00a9ea900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc00a9ea900, {0xc01c9b0930, 0x2b, 0x6d}, 0x13, 0x2f) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc00a9ea900, {0xc02b2de200, 0x2b, 0x6d}, {0xc01c9b0930, 0x0, 0xc00a9ea900}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc00a9ea900, {0xc02b2de200, 0x2b, 0x74}, {0xc01c9b0930, 0x2b, 0x6d}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0929bcd98, {0xc02b2de200, 0x2b, 0x74}, {0xc01c9b0930, 0x2b, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0929bb6c0, {0xc01c9b0930, 0x0, 0xc043b474e0}, 0x121ea30, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0929bb6c0, {0xc02b2de200, 0x2b, 0x6d}, {0xc01c9b0930, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0929bb400, {0xc01c9b0930, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc09ac00688, {{0xc04d61c1e0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0622ba400, {{0xc04d61c1e0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08258a400, {0x63c85b8, 0xc079e521e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc079e521e0}, {0x64a6560, 0xc0622ba400}, {0xc04d61c1e0, 0x0, 0x30}, {0x174301108c6ed5a3, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc079e521e0}, {0x6466ed8, 0xc09ac00680}, {0xc04d61c1e0, 0x2a, 0x30}, {0x174301108c6ed5a3, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc053923a40}, {{0x174301108c6ed5a3, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301108c6ed5a3, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0645eed00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110aa3c3aa3, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01de0d000, {0x63c85b8, 0xc079e521e0}, 0xc0645eed00, 0xc06bdc8640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01de0d000, {0x63c85b8, 0xc079e521e0}, 0xc0645eed00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108c6ed5a3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108c6ed5a3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xe, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079e52150}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079e52150}, {0xc043b4d3f8, 0x660089}, 0xc043b4d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079e52150}, {0x4c3451474e303069}, 0xc0645eeb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079e520c0}, 0xc0645eeb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}, 0xc03ebddc00, 0xc031cfba58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}, 0x3, 0xc015b9f380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043b4d980}}}, {0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}, 0xc03ebddc00, 0xc015b9f380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079e520c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079e520c0}, {0x40, 0xc043b4db70}, 0xc043b4dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079e520c0}, {0x4e9d7a0, 0xc0645eeb00}, 0xc03ebddc00, 0xc031cfba58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079e520c0}, 0xc01c343140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc045414fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc045414fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219793563 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc055e6cd20}, 0xc002ca0cb0, {0x646cc48, 0xc055e6cd80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc055e6cd20}, 0xedb7a834e, 0xc02193c120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc055e6cd20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07a7efa70}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc002ca0ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc07a7efa70}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804878 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc023b91680, {0x63c85b8, 0xc050dd74d0}, 0xc041855cc8, 0x1, 0xc078bb30a8, 0x519bbc, 0xc06f6b9c80, 0xc019fec2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc050dd74d0}, 0x120, 0xc078bb3230, 0x6f6b9c20, 0xc036f001a0, 0x18, 0xc06f6b9c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc023b91680, {0x63c85b8, 0xc050dd74d0}, 0xc06f6b9c20, {{{{0x0, 0x0}, {0xc0517b3560, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc050dd74d0}, 0x6352410, 0x2f62fd98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc023a89590, {0x63c85b8, 0xc050dd74d0}, {0xc06f6b9b00, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc023a89590, {0x63c85b8, 0xc050dd74d0}, 0xc064d9e960, 0xc078bb39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc050dd74d0}, 0x660089, {0xc06f6b9b00, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023bf0800, {0x63c85b8, 0xc050dd74d0}, 0xc0448c3600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050dd7440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050dd7440}, {0xc078bb53f8, 0x660089}, 0xc078bb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050dd7440}, {0x4d939c}, 0xc0448c3400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050dd73b0}, 0xc0448c3400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}, 0xc0664fefe0, 0xc03c2ee0c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}, 0x3, 0xc054bb8dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078bb5980}}}, {0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}, 0xc0664fefe0, 0xc054bb8dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc050dd73b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050dd73b0}, {0x40, 0xc078bb5b70}, 0xc078bb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}, 0x203015, 0x203015) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc050dd73b0}, {0x4e9d7a0, 0xc0448c3400}, 0xc0664fefe0, 0xc03c2ee0c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc050dd73b0}, 0xc04c94a7e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85c6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85c6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219113662 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab979a0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0272da500, 0xc00c697800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0272da500, {0xc00c697800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0272da500, {0xc00c697800, 0x5, 0xc0946acbc8}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc046cec5b0, {0xc00c697800, 0x7eda79, 0x6}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc023f12d80, {0xc00c697800, 0x35318b4, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc043a90000, {0xc00c697800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0b9eee978, {0x631b900, 0xc043a90000}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0b9eee700, {0x7fe42ab59310, 0xc023f12d80}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0b9eee700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0b9eee700, {0xc09e21e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0492ae708, {0xc09e21e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc058ccf588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc058ccf588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc058ccfb60, {0x634aac0, 0xc058ccf588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc058ccf500, 0xc03bf178a0, 0xc0946ad42b, 0xc0946ad42d, {0x635bec0, 0xc02a5fb9e0}, 0xc000d6ae00, 0xc0b0009228, 0xc0946ad429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc058ccf500, {0x63c85b8, 0xc02fb08c30}, 0xc03bf17890, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc04364106d, 0x8}}, 0x0, 0xc02fb08bd0, 0xc02fb08c00, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02fb08c30}, {0x643d810, 0xc023f12d80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07cce0180}, {0x643d810, 0xc023f12d80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07cce0180}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807698 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc06e9a8ba0}, 0xc07d48f368, 0x1, 0xc0056070a8, 0x519bbc, 0xc097447a40, 0xc066b8d260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0122ede60, {0x63c85b8, 0xc06e9a8ba0}, 0xc05c2b4b40, 0xc005607230, 0x97447aa0, 0x0, 0x0, 0xc097447a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc06e9a8ba0}, 0xc0974479e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc05a67d440, 0x64}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06e9a8ba0}, 0x6352260, 0x290b4940) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc06e9a8ba0}, {0xc0974478c0, {0x174301101d2e34ec, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc06e9a8ba0}, 0xc077740780, 0xc0056079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06e9a8ba0}, 0x660089, {0xc0974478c0, {0x174301101d2e34ec, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc06e9a8ba0}, 0xc098d2fa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d2e34ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d2e34ec, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e9a8b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e9a8b10}, {0xc0056093f8, 0x660089}, 0xc005609400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e9a8b10}, {0x7fe38e27e4b0}, 0xc098d2f800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e9a8a20}, 0xc098d2f800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}, 0xc016e3a480, 0xc0765de918) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}, 0x3, 0xc05fde8540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005609980}}}, {0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}, 0xc016e3a480, 0xc05fde8540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06e9a8a20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e9a8a20}, {0x40, 0xc005609b70}, 0xc005609ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06e9a8a20}, {0x4e9d7a0, 0xc098d2f800}, 0xc016e3a480, 0xc0765de918) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06e9a8a20}, 0xc0610260c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0974477a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0974477a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219529177 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc022ab4790, {0x63c85b8, 0xc055865ce0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc08bc4ac00, {0x63c85b8, 0xc055865ce0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc08bc4ac00, {0x63c8510, 0xc06630d200}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc08bc4ac00, {0x63c8510, 0xc06630d200}, {0x63f7db8, 0x9bf6f48}, 0xc08b5b9088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc08bc4ac00, {0x63c85b8, 0xc0b3a56000}, {{0x63f7db8, 0x9bf6f48}, {0xc00ea00840, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc08bc4ac00, {0x63c85b8, 0xc0b3a56000}, {{0x63f7db8, 0x9bf6f48}, {0xc00ea00840, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc00ea00840, 0x6}, 0x0, 0x0}, {0x6a9e47e, 0xedb7a834f, 0x0}, {0x6a9ed6d, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08bc4ac00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08bc4ac00, {0x63c8510, 0xc0285f6d80}, 0xc0a8173dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01f1d6900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0285f6d80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01f1d6900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811316 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc021131aa0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 218905996 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaffb3920, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc025f2a180, 0xc0aa044000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc025f2a180, {0xc0aa044000, 0x6da4, 0x6da4}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc025f2a180, {0xc0aa044000, 0x6da4, 0xc06097cf00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc017bb3810, {0xc0aa044000, 0x7eda79, 0xc0aa044005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01aa2a7e0, {0xc0aa044000, 0x3, 0x6da4}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc018c97f68, {0xc0aa044000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0c2841af8, {0x631b900, 0xc018c97f68}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0c2841880, {0x7fe42ab59310, 0xc01aa2a7e0}, 0x6da4) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0c2841880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0c2841880, {0xc0b8ce3000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0272bc408, {0xc0b8ce3000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc06118a088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc06118a088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc06118a660, {0x634aac0, 0xc06118a088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc06118a000, 0xc053091e70, 0xc09471742b, 0xc09471742d, {0x635bec0, 0xc07bb49950}, 0xc000d6ae00, 0xc0974cba48, 0xc094717429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc06118a000, {0x63c85b8, 0xc0235bcab0}, 0xc053091e60, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0b8cc1037, 0x8}}, 0x0, 0xc0235bc840, 0xc0235bc990, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0235bcab0}, {0x643d810, 0xc01aa2a7e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc039631260}, {0x643d810, 0xc01aa2a7e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc039631260}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219013272 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc067e2c210, 0xb3) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc015566a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0155670e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0e2f25300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0e2f25300, {0x63c8510, 0xc067e2c240}, 0xc0bb425dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0490ff9e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc067e2c240}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0490ff9e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802540 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021defa40, {0x63c85b8, 0xc044416900}, 0xc02ab7ee40, 0x1, 0xc04dcf10a8, 0x519bbc, 0xc049337380, 0xc036ad2ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc044416900}, 0x120, 0xc04dcf1230, 0x49337320, 0x0, 0x660069, 0xc049337380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021defa40, {0x63c85b8, 0xc044416900}, 0xc049337320, {{{{0x0, 0x0}, {0xc023018900, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc044416900}, 0x6352410, 0x65636d18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3df40, {0x63c85b8, 0xc044416900}, {0xc049337200, {0x1743011020e783d2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3df40, {0x63c85b8, 0xc044416900}, 0xc047fe7e00, 0xc04dcf19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044416900}, 0x660089, {0xc049337200, {0x1743011020e783d2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021ec1400, {0x63c85b8, 0xc044416900}, 0xc0115e4b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020e783d2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020e783d2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044416840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044416840}, {0xc04dcf33f8, 0x660089}, 0xc04dcf3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044416840}, {0xc0015221e0}, 0xc0115e4600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0444167b0}, 0xc0115e4600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}, 0xc0968d01a0, 0xc06316f050) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}, 0x3, 0xc056d163c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04dcf3980}}}, {0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}, 0xc0968d01a0, 0xc056d163c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0444167b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0444167b0}, {0x40, 0xc04dcf3b70}, 0xc04dcf3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0444167b0}, {0x4e9d7a0, 0xc0115e4600}, 0xc0968d01a0, 0xc06316f050) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0444167b0}, 0xc05afdca80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc047c806c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc047c806c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219756634 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe7a128, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0081ba000, 0xc0727eb800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0081ba000, {0xc0727eb800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0081ba000, {0xc0727eb800, 0x735, 0xc0509f8a00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc014c74a48, {0xc0727eb800, 0x7eda79, 0xc0727eb805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0aaac31a0, {0xc0727eb800, 0x3, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0434fa978, {0xc0727eb800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc090aa2978, {0x631b900, 0xc0434fa978}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc090aa2700, {0x7fe42ab59310, 0xc0aaac31a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc090aa2700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc090aa2700, {0xc04ee64000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc028724600, {0xc04ee64000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc043400b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc043400b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0434010e0, {0x634aac0, 0xc043400b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc043400a80, 0xc04a69a9e0, 0xc07c79742b, 0xc07c79742d, {0x635bec0, 0xc029b39440}, 0xc000d6ae00, 0xc03319fae0, 0xc07c797429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc043400a80, {0x63c85b8, 0xc06fef5b60}, 0xc04a69a9d0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc07d54d03b, 0x8}}, 0x0, 0xc06fef5b00, 0xc06fef5b30, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06fef5b60}, {0x643d810, 0xc0aaac31a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc052e636e0}, {0x643d810, 0xc0aaac31a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc052e636e0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812614 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc0304ca2d0}, 0xc031cfb080, 0x1, 0xc041e330a8, 0x519bbc, 0xc016d8ec00, 0xc073352c00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05afa9500, {0x63c85b8, 0xc0304ca2d0}, 0xc042915320, 0xc041e331a8, 0x16d8ec00, 0x0, 0x0, 0xc016d8ec00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc0304ca2d0}, 0xc016d8eb40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc02f72e480, 0x53}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0304ca2d0}, 0x6352260, 0x47c52618) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc0304ca2d0}, {0xc016d8e6c0, {0x174301105988909c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc0304ca2d0}, 0xc066746e60, 0xc041e339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0304ca2d0}, 0x660089, {0xc016d8e6c0, {0x174301105988909c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0304ca2d0}, 0xc0460de700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105988909c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105988909c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63bfb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63bfb0}, {0xc041e353f8, 0x660089}, 0xc041e35400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63bfb0}, {0x7fe3ba6966c0}, 0xc0460de400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63bf20}, 0xc0460de400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}, 0xc03ebdd720, 0xc031cfb2f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}, 0x3, 0xc033d33f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc041e35980}}}, {0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}, 0xc03ebdd720, 0xc033d33f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63bf20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63bf20}, {0x40, 0xc041e35b70}, 0xc041e35ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63bf20}, {0x4e9d7a0, 0xc0460de400}, 0xc03ebdd720, 0xc031cfb2f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63bf20}, 0xc039f0b380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812681 [runnable]: sync.runtime_SemacquireMutex(0x0, 0xff, 0xc07cbc23f0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x459e98, 0x0, 0xc0432cee88) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x118663c, 0xc0432ceee0, 0x1194a0b, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc02062d080, {0x29c223a, 0x30eb}, 0x0, 0xc0cc27b8f8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0cc27b680, {0xffffffffffffff15, 0xc0432cf050}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0cc27b680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0cc27b680, {0xc0965227d0, 0x27, 0x4a}, 0x13, 0x13) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekGE(0xc0cc27b680, {0xc0965227d0, 0x27, 0x4a}, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:513 +0xaf github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekGE(0xc0cc27b680, {0xc0965227d0, 0x27, 0x4a}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1317 +0x29d github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc093190140, {0xc0965227d0, 0x27, 0x4a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:409 +0xd6 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09318ecc0, {0xc0965227d0, 0x50, 0x4a}, 0x27, 0x27) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc09318ecc0, {0xc0965227d0, 0xc0965227d0, 0xc096522780}, 0x4a) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc09318ea00, {0xc0965227d0, 0x27, 0x4a}, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/pebble.(*Iterator).SeekGE(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:865 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04272bcd8, {{0xc045f2f020, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:290 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc045600900, {{0xc045f2f020, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc05195d000, {{0xc045f2f020, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc05195d000, {0x63c85b8, 0xc06c832780}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc045f2f050, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.MVCCScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc045f2f050, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2504 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc01e069ae8}, {{0x174301109eb5cab3, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:62 +0x41b github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109eb5cab3, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0650c2a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bc832fb3, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e068000, {0x63c85b8, 0xc06c832780}, 0xc0650c2a00, 0xc043e92a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc06c832780}, 0xc0650c2a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109eb5cab3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109eb5cab3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06c8326f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c8326f0}, {0xc0432d53f8, 0x660089}, 0xc0432d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06c8326f0}, {0xc0015221e0}, 0xc0650c2800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06c832660}, 0xc0650c2800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}, 0xc0bc1da2c0, 0xc05bec8e10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}, 0x3, 0xc005a6d400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0432d5980}}}, {0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}, 0xc0bc1da2c0, 0xc005a6d400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06c832660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c832660}, {0x40, 0xc0432d5b70}, 0xc0432d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06c832660}, {0x4e9d7a0, 0xc0650c2800}, 0xc0bc1da2c0, 0xc05bec8e10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06c832660}, 0xc03112ca20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03d0ccd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03d0ccd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799746 [runnable]: sync.runtime_SemacquireMutex(0xc094d90690, 0x8f, 0x7fd464b291c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc094d90538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc094d90560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0086ad200, {0xffffffffffffff15, 0xc094d906c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0086ad200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0086ad200, {0xc024254280, 0x32, 0x74}, {0xc02320bf80, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc088b59e98, {0xc024254280, 0x32, 0x74}, {0xc02320bf80, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc088b587c0, {0xc02320bf80, 0x77, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc088b587c0, {0xc024254280, 0x77, 0xc02320bf80}, {0xc02320bf80, 0xc067e54ee0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc088b58500, {0xc02320bf80, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc051db6ea8, {{0xc067e54ee0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06ae61800, {{0xc06b6200c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc044dc3000, {0x63c85b8, 0xc09ed74360}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc09ed74360}, {0x64a6560, 0xc06ae61800}, {0xc06b6200c0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc09ed74360}, {0x6466ed8, 0xc051db6c00}, {0xc06b6200c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04459ee58}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc044a74d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0205cd000, {0x63c85b8, 0xc09ed74360}, 0xc044a74d00, 0xc048ab3400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cd000, {0x63c85b8, 0xc09ed74360}, 0xc044a74d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09ed742d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09ed742d0}, {0xc01fad68f0, 0x660089}, 0xc01fad68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09ed742d0}, {0xc00102e4e0}, 0xc044a74c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09ed74270}, 0xc044a74c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc09ed740c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810202 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01edcb400, {0x63c85b8, 0xc0853c3530}, 0xc0cd522800, 0xc04a97d900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0853c3530}, 0xc0cd522800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c3be46b, 0x0, 0x0}, 0xc04e7ebb20, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c3be46b, 0x0, 0x0}, 0xc04e7ebb20, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0853c3440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0853c3440}, {0xc01fdc33f8, 0x660089}, 0xc01fdc3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0853c3440}, {0x40e8faff2ddd4c3a}, 0xc0cd522600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0853c33b0}, 0xc0cd522600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}, 0xc02299c780, 0xc029398d38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}, 0x3, 0xc07b3d5240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fdc3980}}}, {0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}, 0xc02299c780, 0xc07b3d5240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0853c33b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0853c33b0}, {0x40, 0xc01fdc3b70}, 0xc01fdc3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0853c33b0}, {0x4e9d7a0, 0xc0cd522600}, 0xc02299c780, 0xc029398d38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0853c33b0}, 0xc09b2d7bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc073b325a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc073b325a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802988 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc044ecb020}, 0xc0b2e63a88, 0x1, 0xc06c6b50a8, 0x519bbc, 0xc041c50240, 0xc040913aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc044ecb020}, 0x1e0, 0xc06c6b5230, 0x41c501e0, 0x0, 0x660069, 0xc041c50240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc044ecb020}, 0xc041c501e0, {{{{0x0, 0x0}, {0xc007f450e0, 0x1a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x649b418, {0x63c85b8, 0xc044ecb020}, 0x413e3c00b50b0eff, 0x4d337ea3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc044ecb020}, {0xc075f558c0, {0x1743011027f78ab5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc044ecb020}, 0xc0463c1ae0, 0xc06c6b59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044ecb020}, 0x660089, {0xc075f558c0, {0x1743011027f78ab5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc044ecb020}, 0xc0719bf900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044ecaf90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044ecaf90}, {0xc06c6b73f8, 0x660089}, 0xc06c6b7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044ecaf90}, {0x3feffffff1b1a01c}, 0xc0719bf700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044ecaf00}, 0xc0719bf700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}, 0xc061dfa160, 0xc06cf4ad50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}, 0x3, 0xc0a9f8c480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06c6b7980}}}, {0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}, 0xc061dfa160, 0xc0a9f8c480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc044ecaf00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044ecaf00}, {0x40, 0xc06c6b7b70}, 0xc06c6b7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc044ecaf00}, {0x4e9d7a0, 0xc0719bf700}, 0xc061dfa160, 0xc06cf4ad50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc044ecaf00}, 0xc035294b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075f557a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075f557a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812573 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808275 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ad80780, {0x63c85b8, 0xc08a603740}, 0xc07ab15950, 0x1, 0x0, 0x1, 0xc072f77020, 0xc07854c2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08a603740}, 0x120, 0xc00c8ec728, 0x72f76fc0, 0x0, 0x660069, 0xc072f77020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ad80780, {0x63c85b8, 0xc08a603740}, 0xc072f76fc0, {{{{0x0, 0x0}, {0xc01f596360, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08a603740}, 0x6352410, 0x2ad6f600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ad46af0, {0x63c85b8, 0xc08a603740}, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ad46af0, {0x63c85b8, 0xc08a603740}, 0xc0bcc25900, 0xc078182ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08a603740}, 0x660089, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ad9a000, {0x63c85b8, 0xc08a603740}, 0xc07babd800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08a6036b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a6036b0}, {0xc09365a8f0, 0x660089}, 0xc09365a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08a6036b0}, {0x0}, 0xc07babd600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08a603650}, 0xc07babd600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc045e73dd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219782674 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc05a2fe580, {0x63c85b8, 0xc093b417d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc057ebf300, {0x63c85b8, 0xc093b417d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc057ebf300, {0x63c8510, 0xc035a1dc80}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc057ebf300, {0x63c8510, 0xc035a1dc80}, 0xc0766cd940, {0x7fe42b790cf0, 0xc09b0ca6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc057ebf300, {0x63c85b8, 0xc057756a80}, {{0x63f88f8, 0xc0766cd940}, {0xc08d6643b8, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc057ebf300, {0x63c85b8, 0xc057756a80}, {{0x63f88f8, 0xc0766cd940}, {0xc08d6643b8, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0766cd940}, {0xc08d6643b8, 0x23}, 0x0, 0x0}, {0x2dbb9b87, 0xedb7a834f, 0x0}, {0x2dbba1f7, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc057ebf300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc057ebf300, {0x63c8510, 0xc058e7e580}, 0xc04c2f1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08fb83560) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc058e7e580}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08fb83560) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809806 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc01cdd5590}, 0xc0bb3a8030, 0x1, 0xc005dd3670, 0x519bbc, 0xc08979cd80, 0xc05c050c00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0189dcb40, {0x63c85b8, 0xc01cdd5590}, 0xc05b375200, 0xc005dd3770, 0x8979cd80, 0x0, 0x2, 0xc08979cd80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc01cdd5590}, 0xc0195215c0, {{{{0xc079db7b00, 0x46}, {0xc04f02a6c0, 0x64}}, {{0xc082319200, 0x92}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01cdd5590}, 0x63521a0, 0x6b245498) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc01cdd5590}, {0xc04aa28d80, {0x174301100792de8e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc01cdd5590}, 0xc04c7535e0, 0xc005dd3fb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01cdd5590}, 0x660089, {0xc04aa28d80, {0x174301100792de8e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc01cdd5590}, 0xc06f82ef00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100792de8e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100792de8e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01cdd54a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01cdd54a0}, {0xc0781839c0, 0x660089}, 0xc0781839c8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01cdd54a0}, {0xc00102e4e0}, 0xc06f82ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01cdd53b0}, 0xc06f82ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x63c85b8, 0xc01cdd4c90}, {0x7fe422a51230, 0xc06e57e600}, 0xc02b533b00, {0xc04aa28360, 0x6, 0x6}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x24d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x63c85b8, 0xc01cdd4c90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812587 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811054 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ecd9400, {0x63c85b8, 0xc01ac7c0c0}, 0xc04eed6f00, 0xc03c0315e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ecd9400, {0x63c85b8, 0xc01ac7c0c0}, 0xc04eed6f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011076693cdb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011076693cdb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01ac7c030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ac7c030}, {0xc031e9d3f8, 0x660089}, 0xc031e9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01ac7c030}, {0x7fe137a9ce50}, 0xc04eed6c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08d943f80}, 0xc04eed6c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}, 0xc028541fe0, 0xc01c104f00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}, 0x3, 0xc0c8d82c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc031e9d980}}}, {0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}, 0xc028541fe0, 0xc0c8d82c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08d943f80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d943f80}, {0x40, 0xc031e9db70}, 0xc031e9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08d943f80}, {0x4e9d7a0, 0xc04eed6c00}, 0xc028541fe0, 0xc01c104f00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08d943f80}, 0xc04abe3320, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc062b5c240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc062b5c240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803507 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0201e3b80, {0x63c85b8, 0xc06bdf2780}, 0xc0722e42a0, 0x1, 0x0, 0x1, 0xc03a3f3260, 0xc03a3f22a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06bdf2780}, 0x120, 0xc07640e728, 0x3a3f3200, 0x0, 0x660069, 0xc03a3f3260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0201e3b80, {0x63c85b8, 0xc06bdf2780}, 0xc03a3f3200, {{{{0x0, 0x0}, {0xc0435a0c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06bdf2780}, 0x6352410, 0x32cb3bc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01f8c5950, {0x63c85b8, 0xc06bdf2780}, {0xc03a90a480, {0x174301102cffdd45, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01f8c5950, {0x63c85b8, 0xc06bdf2780}, 0xc089013900, 0xc076588ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06bdf2780}, 0x660089, {0xc03a90a480, {0x174301102cffdd45, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020435400, {0x63c85b8, 0xc06bdf2780}, 0xc0571e9e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cffdd45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cffdd45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06bdf26f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06bdf26f0}, {0xc002fea8f0, 0x660089}, 0xc002fea8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06bdf26f0}, {0xc00102e4e0}, 0xc0571e9d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06bdf2690}, 0xc0571e9d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc075bd85a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807300 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc01efb1f90, 0x1e) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc029d90a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc029d910e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc081f30600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc081f30600, {0x63c8510, 0xc01efb1fc0}, 0xc076677dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0924a1250) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc01efb1fc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0924a1250) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801428 [runnable]: sync.runtime_SemacquireMutex(0xc08f4d2690, 0x8f, 0xc08f4d24a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x3, 0xc08f4d2538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc08f4d2560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0828d4d80, {0xffffffffffffff15, 0xc08f4d26c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0828d4d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0828d4d80, {0xc07433e660, 0x32, 0x57}, {0xc02bad58c0, 0x32, 0x58}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc05f5a1998, {0xc07433e660, 0x32, 0x57}, {0xc02bad58c0, 0x32, 0x58}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc05f5a02c0, {0xc02bad58c0, 0x58, 0x57}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc05f5a02c0, {0xc07433e660, 0x58, 0xc02bad58c0}, {0xc02bad58c0, 0xc058094ee0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc05f5a0000, {0xc02bad58c0, 0x32, 0x58}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0660703a8, {{0xc058094ee0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04d14bd00, {{0xc0a1b1faa0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc06b891000, {0x63c85b8, 0xc067188510}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc067188510}, {0x64a6560, 0xc04d14bd00}, {0xc0a1b1faa0, 0x53, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc067188510}, {0x6466ed8, 0xc066070100}, {0xc0a1b1faa0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0198abbf0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06308ce00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c91800, {0x63c85b8, 0xc067188510}, 0xc06308ce00, 0xc071c07720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c91800, {0x63c85b8, 0xc067188510}, 0xc06308ce00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067188480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067188480}, {0xc0936368f0, 0x660089}, 0xc0936368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067188480}, {0xc00102e4e0}, 0xc06308cd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067188420}, 0xc06308cd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc046ae9fb0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811936 [semacquire]: sync.runtime_SemacquireMutex(0xc03b1bc000, 0x3, 0xc03b1bc000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc0050c4b68) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fe21197a000, 0x7e7f, 0x7fdb949c0ec0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc04142d400, 0x7fe21197a001, 0x7fe21197a020, 0x2543, 0x2548) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc0374ac580, {0x15868c1, 0x2543}, 0x0, 0xc066b7ecf8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc066b7ea80, {0xffffffffffffff15, 0xc0050c4df0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc066b7ea80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc066b7ea80, {0xc03ab21c50, 0x27, 0x27}, 0x0, 0x50) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekGE(0xc07752d9e8, {0xc03ab21c50, 0x1, 0x27}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:513 +0xaf github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc07752d9e8, {0xc03ab21c50, 0x27, 0x27}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:409 +0xd6 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07752c7c0, {0xc03ab21c50, 0x30, 0x27}, 0x27, 0x27) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc07752c7c0, {0xc03ab21c50, 0xc03ab21c50, 0xc03ab21c20}, 0x27) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc07752c500, {0xc03ab21c50, 0x27, 0x27}, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/pebble.(*Iterator).SeekGE(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:865 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc042407758, {{0xc03ab21b30, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:290 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0664b8300, {{0xc03ab21b30, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc0529ab000, {{0xc03ab21b30, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc0529ab000, {0x63c85b8, 0xc050b466f0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc03ab21b60, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.mvccScanToKvs({_, _}, {_, _}, {_, _, _}, {0xc03ab21b60, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2318 +0x11e github.com/cockroachdb/cockroach/pkg/storage.MVCCScan({_, _}, {_, _}, {_, _, _}, {0xc03ab21b60, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2491 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc09e287878}, {{0x1743011080c103f0, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:69 +0x5db github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011080c103f0, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0c122d800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174301109e8e68f0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc016a62000, {0x63c85b8, 0xc050b466f0}, 0xc0c122d800, 0xc0307a70e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016a62000, {0x63c85b8, 0xc050b466f0}, 0xc0c122d800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011080c103f0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011080c103f0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050b46660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050b46660}, {0xc0050cb3f8, 0x660089}, 0xc0050cb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050b46660}, {0x7fe42e248098}, 0xc0c122d600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050b465d0}, 0xc0c122d600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}, 0xc03eff2ea0, 0xc0408ce6c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}, 0x3, 0xc039d39740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0050cb980}}}, {0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}, 0xc03eff2ea0, 0xc039d39740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc050b465d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050b465d0}, {0x40, 0xc0050cbb70}, 0xc0050cbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc050b465d0}, {0x4e9d7a0, 0xc0c122d600}, 0xc03eff2ea0, 0xc0408ce6c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc050b465d0}, 0xc0670a90e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05b3c1e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05b3c1e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809757 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc061a9a660}, 0xc045c2c078, 0x1, 0x0, 0x1, 0xc07b860600, 0xc05f409a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc061a9a660}, 0x120, 0xc01abda728, 0x7b8605a0, 0x0, 0x660069, 0xc07b860600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc061a9a660}, 0xc07b8605a0, {{{{0x0, 0x0}, {0xc0262b7440, 0xe8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc061a9a660}, 0x6352410, 0x3f8eb2f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc061a9a660}, {0xc087e06120, {0x1743011066f9ca19, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc061a9a660}, 0xc00854f2c0, 0xc076022ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc061a9a660}, 0x660089, {0xc087e06120, {0x1743011066f9ca19, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc061a9a660}, 0xc0824c4700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066f9ca19, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066f9ca19, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc061a9a570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc061a9a570}, {0xc0076ca8f0, 0x660089}, 0xc0076ca8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc061a9a570}, {0xc00102e4e0}, 0xc0824c4600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc061a9a510}, 0xc0824c4600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04996c630}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803224 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc04967bb00}, 0xc05796a6a8, 0x1, 0xc0472fb0a8, 0x519bbc, 0xc03487eba0, 0xc0328e5260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04967bb00}, 0x120, 0xc0472fb230, 0x3487eb40, 0x0, 0x660069, 0xc03487eba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc04967bb00}, 0xc03487eb40, {{{{0x0, 0x0}, {0xc079ab0240, 0x15}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04967bb00}, 0x6352410, 0x3214b0a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc04967bb00}, {0xc03487ea20, {0x174301102b7eff25, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc04967bb00}, 0xc048083d60, 0xc0472fb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04967bb00}, 0x660089, {0xc03487ea20, {0x174301102b7eff25, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc04967bb00}, 0xc040a2b800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b7eff25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b7eff25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04967ba70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04967ba70}, {0xc0472fd3f8, 0x660089}, 0xc0472fd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04967ba70}, {0x7fe4256582b0}, 0xc040a2b600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04967b9e0}, 0xc040a2b600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}, 0xc0681df780, 0xc02733e660) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}, 0x3, 0xc01bab9640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0472fd980}}}, {0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}, 0xc0681df780, 0xc01bab9640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04967b9e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04967b9e0}, {0x40, 0xc0472fdb70}, 0xc0472fdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04967b9e0}, {0x4e9d7a0, 0xc040a2b600}, 0xc0681df780, 0xc02733e660) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04967b9e0}, 0xc01d0dfa40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03487e900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03487e900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808407 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0249cd8f0}, 0xc08666dc00, 0xc0777415e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0249cd8f0}, 0xc08666dc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c286a81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c286a81, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0249cd860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0249cd860}, {0xc00d96b3f8, 0x660089}, 0xc00d96b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0249cd860}, {0x545446444f533955}, 0xc08666d900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0249cd7d0}, 0xc08666d900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}, 0xc06cadf3e0, 0xc087360c60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}, 0x3, 0xc04fcada40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00d96b980}}}, {0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}, 0xc06cadf3e0, 0xc04fcada40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0249cd7d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0249cd7d0}, {0x40, 0xc00d96bb70}, 0xc00d96bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0249cd7d0}, {0x4e9d7a0, 0xc08666d900}, 0xc06cadf3e0, 0xc087360c60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0249cd7d0}, 0xc0431bb8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0518cc6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0518cc6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802829 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc018c11c30, {0x63c85b8, 0xc06b7d75c0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc06b995900, {0x63c85b8, 0xc06b7d75c0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc06b995900, {0x63c8510, 0xc062a545c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc06b995900, {0x63c8510, 0xc062a545c0}, {0x63f7db8, 0x9bf6f48}, 0xc08b0af088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc06b995900, {0x63c85b8, 0xc02c836840}, {{0x63f7db8, 0x9bf6f48}, {0xc02733127b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc06b995900, {0x63c85b8, 0xc02c836840}, {{0x63f7db8, 0x9bf6f48}, {0xc02733127b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc02733127b, 0x6}, 0x0, 0x0}, {0xb884df9, 0xedb7a834f, 0x0}, {0xb885792, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06b995900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06b995900, {0x63c8510, 0xc06576e940}, 0xc058163dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04038c690) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06576e940}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04038c690) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809876 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc02d781e00}, 0xc00c412e40, 0x1, 0xc00396b0a8, 0x519bbc, 0xc081db74a0, 0xc085ab9c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02d781e00}, 0x120, 0xc00396b230, 0x81db7440, 0x0, 0x660069, 0xc081db74a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc02d781e00}, 0xc081db7440, {{{{0x0, 0x0}, {0xc09ecf0d80, 0x32}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02d781e00}, 0x6352410, 0x603b94d8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc02d781e00}, {0xc081db7320, {0x1743011066f2ec44, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc02d781e00}, 0xc060d80460, 0xc00396b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02d781e00}, 0x660089, {0xc081db7320, {0x1743011066f2ec44, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc02d781e00}, 0xc05a3aca00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066f2ec44, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066f2ec44, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02d781bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d781bf0}, {0xc00396d3f8, 0x660089}, 0xc00396d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02d781bf0}, {0x0}, 0xc05a3ac800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02d781a40}, 0xc05a3ac800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}, 0xc03c8503a0, 0xc00c413ef0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}, 0x3, 0xc083dc8940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00396d980}}}, {0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}, 0xc03c8503a0, 0xc083dc8940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02d781a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d781a40}, {0x40, 0xc00396db70}, 0xc00396dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02d781a40}, {0x4e9d7a0, 0xc05a3ac800}, 0xc03c8503a0, 0xc00c413ef0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02d781a40}, 0xc082a8fce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc081db7200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc081db7200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808628 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc0785180c0}, 0xc05a87e800, 0xc050894460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0785180c0}, 0xc05a87e800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e66fd91, 0x0, 0x0}, 0xc05bbd35b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e66fd91, 0x0, 0x0}, 0xc05bbd35b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078518030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078518030}, {0xc00df733f8, 0x660089}, 0xc00df73400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078518030}, {0x126b7a013b010000}, 0xc05a87e600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0634f7f80}, 0xc05a87e600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}, 0xc00ff19680, 0xc07e1133c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}, 0x3, 0xc0518a3c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00df73980}}}, {0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}, 0xc00ff19680, 0xc0518a3c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0634f7f80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0634f7f80}, {0x40, 0xc00df73b70}, 0xc00df73ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0634f7f80}, {0x4e9d7a0, 0xc05a87e600}, 0xc00ff19680, 0xc07e1133c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0634f7f80}, 0xc073bf2600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0652d4900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0652d4900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803701 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b2400, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05c806a80, 0xc0498be000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05c806a80, {0xc0498be000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05c806a80, {0xc0498be000, 0x735, 0xc05afba000}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc055acf5a0, {0xc0498be000, 0x7eda79, 0xc0498be005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0d0fe58c0, {0xc0498be000, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc054046bb8, {0xc0498be000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0ab51d3f8, {0x631b900, 0xc054046bb8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0ab51d180, {0x7fe42ab59310, 0xc0d0fe58c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0ab51d180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0ab51d180, {0xc0a1845000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0300697d0, {0xc0a1845000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04467ab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04467ab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04467b0e0, {0x634aac0, 0xc04467ab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04467aa80, 0xc0671155b0, 0xc09494942b, 0xc09494942d, {0x635bec0, 0xc03614ca20}, 0xc000d6ae00, 0xc029e8bde8, 0xc094949429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04467aa80, {0x63c85b8, 0xc021a2ab40}, 0xc0671155a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08ab9104d, 0x8}}, 0x0, 0xc021a2aab0, 0xc021a2aae0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc021a2ab40}, {0x643d810, 0xc0d0fe58c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0727e8480}, {0x643d810, 0xc0d0fe58c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0727e8480}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808556 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc06c70c5d0}, 0xc043407080, 0x1, 0xc0086e30a8, 0x519bbc, 0xc0572e27e0, 0xc00fd1fc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06c70c5d0}, 0x1e0, 0xc0086e3230, 0x572e2780, 0x0, 0x660069, 0xc0572e27e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc06c70c5d0}, 0xc0572e2780, {{{{0x0, 0x0}, {0xc0a8bcf200, 0x59}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06c70c5d0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc06c70c5d0}, {0xc066edc120, {0x174301105fca4e76, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc06c70c5d0}, 0xc0314401e0, 0xc0086e39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06c70c5d0}, 0x660089, {0xc066edc120, {0x174301105fca4e76, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc06c70c5d0}, 0xc066f3b300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105fca4e76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105fca4e76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06c70c360}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c70c360}, {0xc0086e53f8, 0x660089}, 0xc0086e5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06c70c360}, {0xc018da2800}, 0xc066f3b100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06c70c2d0}, 0xc066f3b100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}, 0xc006db9740, 0xc04784a888) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}, 0x3, 0xc052db4d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0086e5980}}}, {0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}, 0xc006db9740, 0xc052db4d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06c70c2d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c70c2d0}, {0x40, 0xc0086e5b70}, 0xc0086e5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06c70c2d0}, {0x4e9d7a0, 0xc066f3b100}, 0xc006db9740, 0xc04784a888) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06c70c2d0}, 0xc03d505e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc066edc000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc066edc000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811775 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc04d7f7200}, 0xc0208f1830, 0x1, 0xc01669b0a8, 0x519bbc, 0xc040809c80, 0xc04565d140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04d7f7200}, 0x120, 0xc01669b230, 0x40809c20, 0x0, 0x660069, 0xc040809c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc04d7f7200}, 0xc040809c20, {{{{0x0, 0x0}, {0xc06c58a480, 0x3f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d7f7200}, 0x6352410, 0x11fab520) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc04d7f7200}, {0xc040809b00, {0x174301107adc4512, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc04d7f7200}, 0xc04ebc9040, 0xc01669b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d7f7200}, 0x660089, {0xc040809b00, {0x174301107adc4512, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc04d7f7200}, 0xc04a724400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107adc4512, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107adc4512, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d7f7170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d7f7170}, {0xc01669d3f8, 0x660089}, 0xc01669d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d7f7170}, {0xc0015221e0}, 0xc04a724200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d7f70e0}, 0xc04a724200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}, 0xc029ae97a0, 0xc0208f1b90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}, 0x3, 0xc05aca18c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01669d980}}}, {0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}, 0xc029ae97a0, 0xc05aca18c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d7f70e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d7f70e0}, {0x40, 0xc01669db70}, 0xc01669dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d7f70e0}, {0x4e9d7a0, 0xc04a724200}, 0xc029ae97a0, 0xc0208f1b90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d7f70e0}, 0xc025313800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0408099e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0408099e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804857 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01ef69cc0, {0x63c85b8, 0xc077cd1d10}, 0xc031bebb30, 0x1, 0xc079f65ff0, 0x519bbc, 0xc055b20240, 0xc0473598c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc077cd1d10}, 0x1e0, 0xc079f66178, 0x55b201e0, 0xc03b1bda00, 0x18, 0xc055b20240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01ef69cc0, {0x63c85b8, 0xc077cd1d10}, 0xc055b201e0, {{{{0x0, 0x0}, {0xc0b5f80480, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc077cd1d10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01ef6cdc0, {0x63c85b8, 0xc077cd1d10}, {0xc06eb5ab40, {0x174301103269927a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01ef6cdc0, {0x63c85b8, 0xc077cd1d10}, 0xc052ea7220, 0xc064c36938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077cd1d10}, 0x660089, {0xc06eb5ab40, {0x174301103269927a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f030400, {0x63c85b8, 0xc077cd1d10}, 0xc08ca36e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103269927a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103269927a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077cd1c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077cd1c50}, {0xc064c38340, 0x660089}, 0xc064c38348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077cd1c50}, {0xc00102e4e0}, 0xc08ca36d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077cd1bf0}, 0xc08ca36d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0164cbd10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803751 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminTransferLease(0xc02db26000, {0x63c8580, 0xc02cd8b260}, 0x63c8580) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:900 +0x26d github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicateQueue).transferLease(0xc00d506a00, {0x63c8580, 0xc02cd8b260}, 0xc02db26000, {0x0, 0x0, 0x0, 0x0}, 0x300000000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replicate_queue.go:1586 +0x105 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicateQueue).shedLease(0xc00d506a00, {0x63c8580, 0xc02cd8b260}, 0xc02db26000, 0x0, {0x8000000, 0x20000000, {0x15f90, {0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replicate_queue.go:1575 +0x1bb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicateQueue).considerRebalance(0xc00d506a00, {0x63c8580, 0xc02cd8b260}, 0xc02db26000, {0xc02daffbc0, 0x17, 0x4}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replicate_queue.go:1376 +0x976 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicateQueue).processOneChange(0xc00d506a00, {0x63c8580, 0xc02cd8b260}, 0x3b9aca00, 0x4000000000000000, 0x5, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replicate_queue.go:675 +0x1028 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicateQueue).process(0xc00d506a00, {0x63c8580, 0xc02cd8b260}, 0x63c8580, {0x7fe422a51b20, 0xc007d818c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replicate_queue.go:496 +0x305 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica.func1({0x63c8580, 0xc02cd8b260}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:977 +0x26e github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc045e0e1b0}, {0xc08b510510, 0x25}, 0x12a05f2000, 0xc0d6833e28) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica(0xc00d506b40, {0x63c85b8, 0xc045e0e180}, {0x6463f50, 0xc02db26000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:936 +0x3f5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2.1({0x63c85b8, 0xc045e0e180}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:855 +0xde github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812645 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219782119 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc00abbfc30, {0x63c85b8, 0xc04962e3c0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc054d24600, {0x63c85b8, 0xc04962e3c0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc054d24600, {0x63c8510, 0xc0486a1a00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc054d24600, {0x63c8510, 0xc0486a1a00}, 0xc07b991b40, {0x7fe42b790cf0, 0xc001083140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc054d24600, {0x63c85b8, 0xc026d7e600}, {{0x63f88f8, 0xc07b991b40}, {0xc007365bf2, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc054d24600, {0x63c85b8, 0xc026d7e600}, {{0x63f88f8, 0xc07b991b40}, {0xc007365bf2, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc07b991b40}, {0xc007365bf2, 0x23}, 0x0, 0x0}, {0xa9237d5, 0xedb7a8350, 0x0}, {0xa923d37, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc054d24600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc054d24600, {0x63c8510, 0xc053d93d40}, 0xc0bbce9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a843cf0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc053d93d40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a843cf0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219806676 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc08e0d3a10}, 0xc06f265aa0, 0x1, 0xc002cf50a8, 0x519bbc, 0xc049fd7da0, 0xc00c800960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08e0d3a10}, 0x120, 0xc002cf5230, 0x49fd7d40, 0x0, 0x660069, 0xc049fd7da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc08e0d3a10}, 0xc049fd7d40, {{{{0x0, 0x0}, {0xc093ef9680, 0x111}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08e0d3a10}, 0x6352410, 0x5449ae68) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc08e0d3a10}, {0xc049fd7c20, {0x17430110506879a9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc08e0d3a10}, 0xc068771c20, 0xc002cf59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08e0d3a10}, 0x660089, {0xc049fd7c20, {0x17430110506879a9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc08e0d3a10}, 0xc066039800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110506879a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110506879a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08e0d3980}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08e0d3980}, {0xc002cf73f8, 0x660089}, 0xc002cf7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08e0d3980}, {0xc0015221e0}, 0xc066039600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08e0d38f0}, 0xc066039600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}, 0xc08fb7c020, 0xc02ed836c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}, 0x3, 0xc08c740a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002cf7980}}}, {0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}, 0xc08fb7c020, 0xc08c740a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08e0d38f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08e0d38f0}, {0x40, 0xc002cf7b70}, 0xc002cf7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08e0d38f0}, {0x4e9d7a0, 0xc066039600}, 0xc08fb7c020, 0xc02ed836c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08e0d38f0}, 0xc07b8cdc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc049fd7b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc049fd7b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809796 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc012027400, {0x63c85b8, 0xc01dfe4210}, 0xc062d45600, 0xc039c6d400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc012027400, {0x63c85b8, 0xc01dfe4210}, 0xc062d45600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066f28f8f, 0x0, 0x0}, 0xc02426aac0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066f28f8f, 0x0, 0x0}, 0xc02426aac0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01dfe4150}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01dfe4150}, {0xc006c1f3f8, 0x660089}, 0xc006c1f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01dfe4150}, {0x3}, 0xc062d45400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01dfe4090}, 0xc062d45400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}, 0xc060bfb5c0, 0xc02b710a68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}, 0x3, 0xc060842680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006c1f980}}}, {0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}, 0xc060bfb5c0, 0xc060842680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01dfe4090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01dfe4090}, {0x40, 0xc006c1fb70}, 0xc006c1fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01dfe4090}, {0x4e9d7a0, 0xc062d45400}, 0xc060bfb5c0, 0xc02b710a68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01dfe4090}, 0xc07c0d4960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07abd3320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07abd3320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806032 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc08a4e7f80}, 0xc01c98a300, 0x1, 0xc002c330a8, 0x519bbc, 0xc092a39320, 0xc06dbcef60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc08a4e7f80}, 0x1e0, 0xc002c33230, 0x92a392c0, 0x0, 0x660069, 0xc092a39320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc08a4e7f80}, 0xc092a392c0, {{{{0x0, 0x0}, {0xc099e946c0, 0x110}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc08a4e7f80}, 0x203023, 0x203023) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc08a4e7f80}, {0xc04b9d5680, {0x174301104e8f5ee5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc08a4e7f80}, 0xc02f77f180, 0xc002c339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08a4e7f80}, 0x660089, {0xc04b9d5680, {0x174301104e8f5ee5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc08a4e7f80}, 0xc0675ad500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e8f5ee5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e8f5ee5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08a4e7ef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a4e7ef0}, {0xc002c353f8, 0x660089}, 0xc002c35400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08a4e7ef0}, {0xc0015221e0}, 0xc0675ad200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08a4e7e60}, 0xc0675ad200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}, 0xc08623ef20, 0xc01c98bed8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}, 0x3, 0xc08d4a4840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002c35980}}}, {0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}, 0xc08623ef20, 0xc08d4a4840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08a4e7e60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a4e7e60}, {0x40, 0xc002c35b70}, 0xc002c35ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08a4e7e60}, {0x4e9d7a0, 0xc0675ad200}, 0xc08623ef20, 0xc01c98bed8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08a4e7e60}, 0xc093955b00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04b9d5560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04b9d5560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809228 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc03c105dd0}, 0xc07b936f90, 0x1, 0xc01d5450a8, 0x519bbc, 0xc052b78cc0, 0xc08792ecc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03c105dd0}, 0x120, 0xc01d545230, 0x52b78c60, 0x0, 0x660069, 0xc052b78cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc03c105dd0}, 0xc052b78c60, {{{{0x0, 0x0}, {0xc06e108360, 0x16}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03c105dd0}, 0x6352410, 0x450cb020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc03c105dd0}, {0xc052b78b40, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc03c105dd0}, 0xc0316337c0, 0xc01d5459f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03c105dd0}, 0x660089, {0xc052b78b40, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc03c105dd0}, 0xc01a525700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c105d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c105d40}, {0xc01d5473f8, 0x660089}, 0xc01d547400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c105d40}, {0x39128fe2251a4e0a}, 0xc09b41be00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c105cb0}, 0xc09b41be00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}, 0xc079f16a80, 0xc056a75608) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}, 0x3, 0xc085a33800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01d547980}}}, {0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}, 0xc079f16a80, 0xc085a33800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c105cb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c105cb0}, {0x40, 0xc01d547b70}, 0xc01d547ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c105cb0}, {0x4e9d7a0, 0xc09b41be00}, 0xc079f16a80, 0xc056a75608) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c105cb0}, 0xc048a5ba40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330a900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330a900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805750 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc012025000, {0x63c85b8, 0xc083525110}, 0xc04f744900, 0xc0831fe6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc012025000, {0x63c85b8, 0xc083525110}, 0xc04f744900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104806bcf4, 0x0, 0x0}, 0xc02816aaa0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104806bcf4, 0x0, 0x0}, 0xc02816aaa0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1a, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc083525080}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc083525080}, {0xc004e593f8, 0x660089}, 0xc004e59400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc083525080}, {0x8e3c6058023786dc}, 0xc04f744500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc083524ff0}, 0xc04f744500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}, 0xc06f2379c0, 0xc026b79b90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}, 0x3, 0xc06e0fe600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004e59980}}}, {0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}, 0xc06f2379c0, 0xc06e0fe600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc083524ff0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc083524ff0}, {0x40, 0xc004e59b70}, 0xc004e59ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc083524ff0}, {0x4e9d7a0, 0xc04f744500}, 0xc06f2379c0, 0xc026b79b90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc083524ff0}, 0xc044d348a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093732360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093732360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801335 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b078c00, {0x63c85b8, 0xc037af76e0}, 0xc06549e500, 0xc0516259a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b078c00, {0x63c85b8, 0xc037af76e0}, 0xc06549e500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ac1d9c3, 0x0, 0x0}, 0xc04afb8d90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ac1d9c3, 0x0, 0x0}, 0xc04afb8d90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc037af7650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037af7650}, {0xc04d3a73f8, 0x660089}, 0xc04d3a7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc037af7650}, {0x413134343480f0fa}, 0xc06549e300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc037af75c0}, 0xc06549e300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}, 0xc02673dc40, 0xc01d7e2cf0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}, 0x3, 0xc0c63d2b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04d3a7980}}}, {0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}, 0xc02673dc40, 0xc0c63d2b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc037af75c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037af75c0}, {0x40, 0xc04d3a7b70}, 0xc04d3a7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc037af75c0}, {0x4e9d7a0, 0xc06549e300}, 0xc02673dc40, 0xc01d7e2cf0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc037af75c0}, 0xc0c8002cc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc017b07e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc017b07e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808125 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0713b3d50, {0x63c85b8, 0xc0124107b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0124107b0}, {0x4f67ad2, 0x10beec0}, 0xc016471fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219808308 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc05aaa61e0}, 0xc036e9f188, 0x1, 0x0, 0x1, 0xc060e6e960, 0xc07c60c060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05aaa61e0}, 0x120, 0xc00581e728, 0x60e6e900, 0x0, 0x660069, 0xc060e6e960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc05aaa61e0}, 0xc060e6e900, {{{{0x0, 0x0}, {0xc08d00cfc0, 0x2a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05aaa61e0}, 0x6352410, 0x6b2445a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc05aaa61e0}, {0xc07c17a7e0, {0x174301105b1ef448, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc05aaa61e0}, 0xc06810ed20, 0xc07cdf4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05aaa61e0}, 0x660089, {0xc07c17a7e0, {0x174301105b1ef448, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc05aaa61e0}, 0xc051d74600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b1ef448, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b1ef448, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05aaa6150}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05aaa6150}, {0xc0936388f0, 0x660089}, 0xc0936388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05aaa6150}, {0xc00102e4e0}, 0xc051d74500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05aaa60c0}, 0xc051d74500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0782fa150}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806692 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc078683c80}, 0xc02febeac8, 0x1, 0xc0052af0a8, 0x519bbc, 0xc05b0a4060, 0xc0402a74a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc078683c80}, 0x120, 0xc0052af230, 0x5b0a4000, 0x0, 0x660069, 0xc05b0a4060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc078683c80}, 0xc05b0a4000, {{{{0x0, 0x0}, {0xc028ef4240, 0x116}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc078683c80}, 0x6352410, 0x3a07f4f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc078683c80}, {0xc09c253e60, {0x17430110559097be, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc078683c80}, 0xc071dac500, 0xc0052af9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc078683c80}, 0x660089, {0xc09c253e60, {0x17430110559097be, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc078683c80}, 0xc089078700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110559097be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110559097be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078683bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078683bf0}, {0xc0052b13f8, 0x660089}, 0xc0052b1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078683bf0}, {0xc0015221e0}, 0xc089078500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078683b60}, 0xc089078500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}, 0xc08ef7cd40, 0xc05a388c00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}, 0x3, 0xc012589580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0052b1980}}}, {0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}, 0xc08ef7cd40, 0xc012589580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc078683b60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078683b60}, {0x40, 0xc0052b1b70}, 0xc0052b1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc078683b60}, {0x4e9d7a0, 0xc089078500}, 0xc08ef7cd40, 0xc05a388c00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc078683b60}, 0xc0927ca5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09c253d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09c253d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808929 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc057c49b30}, 0xc010decc00, 0xc05ad9fc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc057c49b30}, 0xc010decc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110617803ba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110617803ba, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057c49a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057c49a40}, {0xc00b7193f8, 0x660089}, 0xc00b719400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057c49a40}, {0xc0a1ffc29090cc08}, 0xc010deca00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057c499b0}, 0xc010deca00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}, 0xc0410b86a0, 0xc0390db950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}, 0x3, 0xc016996ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b719980}}}, {0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}, 0xc0410b86a0, 0xc016996ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc057c499b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057c499b0}, {0x40, 0xc00b719b70}, 0xc00b719ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc057c499b0}, {0x4e9d7a0, 0xc010deca00}, 0xc0410b86a0, 0xc0390db950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc057c499b0}, 0xc0465fcd80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08792f0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08792f0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799216 [runnable]: sync.runtime_SemacquireMutex(0xc089f1c690, 0x8f, 0x7fde63317040) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc089f1c538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc089f1c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc043387680, {0xffffffffffffff15, 0xc089f1c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc043387680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc043387680, {0xc030ad4780, 0x32, 0x80}, {0xc0bc75ec80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc087ea4398, {0xc030ad4780, 0x32, 0x80}, {0xc0bc75ec80, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc087ea2cc0, {0xc0bc75ec80, 0x74, 0x80}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc087ea2cc0, {0xc030ad4780, 0x74, 0xc0bc75ec80}, {0xc0bc75ec80, 0xc0616f52b0, 0x31}, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc087ea2a00, {0xc0bc75ec80, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0580c6ea8, {{0xc0616f52b0, 0x31, 0xd0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0049e5600, {{0xc064074b10, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc059032000, {0x63c85b8, 0xc0aea90240}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0aea90240}, {0x64a6560, 0xc0049e5600}, {0xc064074b10, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0aea90240}, {0x6466ed8, 0xc0580c6c00}, {0xc064074b10, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c77e468}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05a267800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0207d7c00, {0x63c85b8, 0xc0aea90240}, 0xc05a267800, 0xc07c2d70e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0207d7c00, {0x63c85b8, 0xc0aea90240}, 0xc05a267800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0aea901b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0aea901b0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0aea901b0}, {0xc00102e4e0}, 0xc05a267700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0aea90150}, 0xc05a267700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc78e70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809661 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e284140, {0x63c85b8, 0xc04c3f51a0}, 0xc05ba09ce0, 0x1, 0x0, 0x1, 0xc077452180, 0xc05c7f6840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04c3f51a0}, 0x120, 0xc026cdc728, 0x77452120, 0x0, 0x660069, 0xc077452180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e284140, {0x63c85b8, 0xc04c3f51a0}, 0xc077452120, {{{{0x0, 0x0}, {0xc040edf560, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04c3f51a0}, 0x6352410, 0x6b245100) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dd81e00, {0x63c85b8, 0xc04c3f51a0}, {0xc046636900, {0x1743011065ed6dda, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dd81e00, {0x63c85b8, 0xc04c3f51a0}, 0xc026033ea0, 0xc07e20eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c3f51a0}, 0x660089, {0xc046636900, {0x1743011065ed6dda, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e28ec00, {0x63c85b8, 0xc04c3f51a0}, 0xc055c90200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ed6dda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ed6dda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c3f5110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c3f5110}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c3f5110}, {0xc00102e4e0}, 0xc055c90100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c3f50b0}, 0xc055c90100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc073da4bd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 216483492 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea2c80, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc060477a80, 0xc028cbd000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc060477a80, {0xc028cbd000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc060477a80, {0xc028cbd000, 0x0, 0x0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc049e65630, {0xc028cbd000, 0xc0, 0x0}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0203b8ae0, {0xc028cbd000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc01bb1eab0, {0xc028cbd000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0036c9e78, {0x631b900, 0xc01bb1eab0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0036c9c00, {0x7fe42ab59310, 0xc0203b8ae0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0036c9c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0036c9c00, {0xc08c8f7000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02ec042a0, {0xc08c8f7000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc026185588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc026185588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc026185b60, {0x634aac0, 0xc026185588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc026185500, 0xc043900c30, 0xc07309b42b, 0xc07309b42d, {0x635bec0, 0xc014fe4f30}, 0xc000d6ae00, 0xc01696b100, 0xc07309b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc026185500, {0x63c85b8, 0xc00dea02a0}, 0xc043900c20, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc07546d025, 0x8}}, 0x0, 0xc00dea0120, 0xc00dea0150, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc00dea02a0}, {0x643d810, 0xc0203b8ae0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0421bbec0}, {0x643d810, 0xc0203b8ae0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0421bbec0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812699 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803975 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc07e1baf90}, 0xc07e119098, 0x1, 0xc0743d50a8, 0x519bbc, 0xc052a88060, 0xc04f1f2a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07e1baf90}, 0x120, 0xc0743d5230, 0x52a88000, 0x0, 0x660069, 0xc052a88060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc07e1baf90}, 0xc052a88000, {{{{0x0, 0x0}, {0xc03b87dd40, 0x1f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07e1baf90}, 0x6352410, 0x55acfbf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc07e1baf90}, {0xc04d339e60, {0x174301103094392a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc07e1baf90}, 0xc017642b40, 0xc0743d59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07e1baf90}, 0x660089, {0xc04d339e60, {0x174301103094392a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc07e1baf90}, 0xc097368700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07e1baf00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e1baf00}, {0xc0743d73f8, 0x660089}, 0xc0743d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07e1baf00}, {0x7fe3f32c3070}, 0xc097368500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07e1bae70}, 0xc097368500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}, 0xc0606eb740, 0xc038313f68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}, 0x3, 0xc02661cb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0743d7980}}}, {0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}, 0xc0606eb740, 0xc02661cb80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07e1bae70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e1bae70}, {0x40, 0xc0743d7b70}, 0xc0743d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07e1bae70}, {0x4e9d7a0, 0xc097368500}, 0xc0606eb740, 0xc038313f68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07e1bae70}, 0xc036292ea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050cffb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050cffb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808405 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc05feaa1d0, {0x63c85b8, 0xc0249cc000}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0249cc000}, {0x4f67ad2, 0x6446ff8}, 0xc01f3f2fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219812245 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0302e5000, {0x63c85b8, 0xc078e82510}, 0xc079a54500, 0xc05f536aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0302e5000, {0x63c85b8, 0xc078e82510}, 0xc079a54500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107f2561ee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107f2561ee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078e82480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078e82480}, {0xc0386373f8, 0x660089}, 0xc038637400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078e82480}, {0x7fe3b8b34988}, 0xc079a54300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078e82390}, 0xc079a54300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}, 0xc0413b6c40, 0xc030c9c2e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}, 0x3, 0xc07b172300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc038637980}}}, {0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}, 0xc0413b6c40, 0xc07b172300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc078e82390}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078e82390}, {0x40, 0xc038637b70}, 0xc038637ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc078e82390}, {0x4e9d7a0, 0xc079a54300}, 0xc0413b6c40, 0xc030c9c2e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc078e82390}, 0xc025c7c600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00c9477a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00c9477a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801017 [runnable]: sync.runtime_SemacquireMutex(0x7fe2db944c40, 0x30, 0x1184940) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0xc010c71380, 0xc0966ae408, 0x4ba25b) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x4571d1, 0x424a070, 0xc0966ae478, 0x28) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc025806840, {0x1bd7a7b, 0x3eca}, 0x0, 0xc0b5a9fe78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0b5a9fc00, {0xffffffffffffff15, 0xc0966ae5e0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0b5a9fc00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0b5a9fc00, {0xc003576c60, 0x2b, 0x59}, 0xc07aa5af00, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0544d2a80, {0xc02afda400, 0xc003576c60, 0x2b}, {0xc003576c60, 0x2b, 0x59}, 0x0, 0xaf) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07aa5ad98, {0xc02afda400, 0x1, 0x59}, {0xc003576c60, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07aa5ad98, {0xc02afda400, 0x2b, 0x73}, {0xc003576c60, 0x2b, 0x59}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07aa596c0, {0xc003576c60, 0x59, 0x73}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07aa596c0, {0xc02afda400, 0x2b, 0xc003576c60}, {0xc003576c60, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07aa59400, {0xc003576c60, 0x2b, 0x59}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06c56ab08, {{0xc08dc6aed0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0634c2400, {{0xc08dc6aed0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0541f4000, {0x63c85b8, 0xc076451140}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc076451140}, {0x64a6560, 0xc0634c2400}, {0xc08dc6aed0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc076451140}, {0x6466ed8, 0xc06c56ab00}, {0xc08dc6aed0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc041282360}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc070acc300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c21800, {0x63c85b8, 0xc076451140}, 0xc070acc300, 0xc064c51360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c21800, {0x63c85b8, 0xc076451140}, 0xc070acc300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0764510b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0764510b0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0764510b0}, {0xc00102e4e0}, 0xc070acc200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076451020}, 0xc070acc200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a4d80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812760 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802674 [runnable]: sync.runtime_SemacquireMutex(0xc079370690, 0x8f, 0x7fe2521f7c00) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc079370538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc079370560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04f6e2d80, {0xffffffffffffff15, 0xc0793706c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc04f6e2d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04f6e2d80, {0xc02d3c9d00, 0x32, 0x79}, {0xc01932df80, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092e8d998, {0xc02d3c9d00, 0x32, 0x79}, {0xc01932df80, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092e8c2c0, {0xc01932df80, 0x32, 0x79}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092e8c2c0, {0xc02d3c9d00, 0x32, 0xc01932df80}, {0xc01932df80, 0xc0cee98540, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092e8c000, {0xc01932df80, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc052d42828, {{0xc0cee98540, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc058452300, {{0xc00a771830, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05315d400, {0x63c85b8, 0xc0c78b55c0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0c78b55c0}, {0x64a6560, 0xc058452300}, {0xc00a771830, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0c78b55c0}, {0x6466ed8, 0xc052d42580}, {0xc00a771830, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc054196b88}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0a7741600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4b800, {0x63c85b8, 0xc0c78b55c0}, 0xc0a7741600, 0xc0394280a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4b800, {0x63c85b8, 0xc0c78b55c0}, 0xc0a7741600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c78b5530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c78b5530}, {0xc0936948f0, 0x660089}, 0xc0936948f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c78b5530}, {0xc00102e4e0}, 0xc0a7741500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c78b54d0}, 0xc0a7741500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220c90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812783 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810966 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04eb1a5d0}, 0xc017535d40, 0x1, 0xc00dcf10a8, 0x519bbc, 0xc039318d80, 0xc0937329c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc089ab4120, {0x63c85b8, 0xc04eb1a5d0}, 0xc07afbe480, 0xc00dcf11a8, 0x39318d80, 0x0, 0x2, 0xc039318d80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04eb1a5d0}, 0xc031460000, {{{{0xc03c329b00, 0x48}, {0xc02e6966c0, 0x90}}, {{0xc045559d40, 0xa8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04eb1a5d0}, 0x63521a0, 0x7b229aa8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04eb1a5d0}, {0xc07bf28480, {0x174301100b0c499a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04eb1a5d0}, 0xc0838c1f40, 0xc00dcf19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04eb1a5d0}, 0x660089, {0xc07bf28480, {0x174301100b0c499a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04eb1a5d0}, 0xc06304fd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100b0c499a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100b0c499a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04eb1a540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04eb1a540}, {0xc00dcf33f8, 0x660089}, 0xc00dcf3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04eb1a540}, {0x7fe137b0cc28}, 0xc06304ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04eb1a480}, 0xc06304ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}, 0xc046910540, 0xc0267d4e70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}, 0x3, 0xc0255425c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00dcf3980}}}, {0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}, 0xc046910540, 0xc0255425c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04eb1a480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04eb1a480}, {0x40, 0xc00dcf3b70}, 0xc00dcf3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04eb1a480}, {0x4e9d7a0, 0xc06304ea00}, 0xc046910540, 0xc0267d4e70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04eb1a480}, 0xc03eb35e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07bf28360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07bf28360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812518 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802611 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc01204f200}, 0xc0a3b12100, 0xc071c060a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01204f200}, 0xc0a3b12100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110229b70e2, 0x0, 0x0}, 0xc02b6ecfb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110229b70e2, 0x0, 0x0}, 0xc02b6ecfb0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01204f170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01204f170}, {0xc0573a73f8, 0x660089}, 0xc0573a7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01204f170}, {0x7fe42930c900}, 0xc00137bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01204f080}, 0xc00137bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}, 0xc03d426580, 0xc0440f6f60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}, 0x3, 0xc06d6c7b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0573a7980}}}, {0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}, 0xc03d426580, 0xc06d6c7b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01204f080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01204f080}, {0x40, 0xc0573a7b70}, 0xc0573a7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01204f080}, {0x4e9d7a0, 0xc00137bf00}, 0xc03d426580, 0xc0440f6f60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01204f080}, 0xc023f13140, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0906e47e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0906e47e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802986 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaaa00, {0x63c85b8, 0xc0737579e0}, 0xc03e3d0d68, 0x1, 0xc06b5650a8, 0x519bbc, 0xc042292060, 0xc041c50060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0737579e0}, 0x1e0, 0xc06b565230, 0x42292000, 0x0, 0x660069, 0xc042292060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaaa00, {0x63c85b8, 0xc0737579e0}, 0xc042292000, {{{{0x0, 0x0}, {0xc01e2039e0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x1, {0x63c85b8, 0xc0737579e0}, 0xa00000, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e7014a0, {0x63c85b8, 0xc0737579e0}, {0xc075108900, {0x1743011027f78ab5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e7014a0, {0x63c85b8, 0xc0737579e0}, 0xc0658914a0, 0xc06b5659f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0737579e0}, 0x660089, {0xc075108900, {0x1743011027f78ab5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ede6400, {0x63c85b8, 0xc0737579e0}, 0xc065905200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073757950}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073757950}, {0xc06b5673f8, 0x660089}, 0xc06b567400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073757950}, {0xc0295c0180}, 0xc065905000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0737578c0}, 0xc065905000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}, 0xc08c4b1120, 0xc083c38228) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}, 0x3, 0xc01be38c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06b567980}}}, {0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}, 0xc08c4b1120, 0xc01be38c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0737578c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0737578c0}, {0x40, 0xc06b567b70}, 0xc06b567ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0737578c0}, {0x4e9d7a0, 0xc065905000}, 0xc08c4b1120, 0xc083c38228) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0737578c0}, 0xc0413fd080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075f55560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075f55560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219755536 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafee9f10, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0843a5680, 0xc070482800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0843a5680, {0xc070482800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0843a5680, {0xc070482800, 0x735, 0xc057a70b40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01f8e15e0, {0xc070482800, 0x7eda79, 0xc070482805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc064072060, {0xc070482800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc023415338, {0xc070482800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06bf97778, {0x631b900, 0xc023415338}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06bf97500, {0x7fe42ab59310, 0xc064072060}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06bf97500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06bf97500, {0xc057b2e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0a74193c8, {0xc057b2e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0a3779588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0a3779588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0a3779b60, {0x634aac0, 0xc0a3779588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0a3779500, 0xc04f0daa30, 0xc0549fd42b, 0xc0549fd42d, {0x635bec0, 0xc04b6594d0}, 0xc000d6ae00, 0xc0395db108, 0xc0549fd429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0a3779500, {0x63c85b8, 0xc0389fe780}, 0xc04f0daa20, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc066f4003b, 0x8}}, 0x0, 0xc0389fe720, 0xc0389fe750, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0389fe780}, {0x643d810, 0xc064072060}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0756ec300}, {0x643d810, 0xc064072060}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0756ec300}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811261 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0623cc420, {0x63c85b8, 0xc04c4cc480}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc055932600, {0x63c85b8, 0xc04c4cc480}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc055932600, {0x63c8510, 0xc07810da00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc055932600, {0x63c8510, 0xc07810da00}, 0xc0305ff610, {0x7fe42b790cf0, 0xc048c51bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc055932600, {0x63c85b8, 0xc03e596f30}, {{0x63f88f8, 0xc0305ff610}, {0xc04a14e927, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc055932600, {0x63c85b8, 0xc03e596f30}, {{0x63f88f8, 0xc0305ff610}, {0xc04a14e927, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0305ff610}, {0xc04a14e927, 0x23}, 0x0, 0x0}, {0x28051636, 0xedb7a8350, 0x0}, {0x28051d21, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc055932600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc055932600, {0x63c8510, 0xc036b2a5c0}, 0xc07d27bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07603a5e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc036b2a5c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07603a5e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811524 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc05bfbe360}, 0xc025186858, 0x1, 0x7080181504dbfe, 0x37e11d600, 0xc06beafda0, 0xc05ed4f260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05bfbe360}, 0x120, 0xc03288f230, 0x6beafd40, 0x0, 0x660069, 0xc06beafda0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc05bfbe360}, 0xc06beafd40, {{{{0x0, 0x0}, {0xc079534d80, 0x30}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05bfbe360}, 0x6352410, 0x33df4570) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc05bfbe360}, {0xc06beafc20, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc05bfbe360}, 0xc066c92780, 0xc03288f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05bfbe360}, 0x660089, {0xc06beafc20, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc05bfbe360}, 0xc05758cd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05bfbe210}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05bfbe210}, {0xc0328913f8, 0x660089}, 0xc032891400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05bfbe210}, {0xc0015221e0}, 0xc05758cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05bfbe180}, 0xc05758cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}, 0xc019167880, 0xc025186828) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}, 0x3, 0xc00dd3c580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032891980}}}, {0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}, 0xc019167880, 0xc00dd3c580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05bfbe180}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05bfbe180}, {0x40, 0xc032891b70}, 0xc032891ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05bfbe180}, {0x4e9d7a0, 0xc05758cb00}, 0xc019167880, 0xc025186828) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05bfbe180}, 0xc03148f5c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06beafb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06beafb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810152 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0181f8c80, {0x63c85b8, 0xc056359dd0}, 0xc046ebef48, 0x1, 0xc01a7f10a8, 0x519bbc, 0xc05a72ade0, 0xc052a10e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc056359dd0}, 0x120, 0xc01a7f1230, 0x5a72ad80, 0x0, 0x660069, 0xc05a72ade0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0181f8c80, {0x63c85b8, 0xc056359dd0}, 0xc05a72ad80, {{{{0x0, 0x0}, {0xc05eb8efc0, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc056359dd0}, 0x6352230, 0x43391eb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0181de7d0, {0x63c85b8, 0xc056359dd0}, {0xc05a72ac60, {0x174301106bac1d71, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0181de7d0, {0x63c85b8, 0xc056359dd0}, 0xc052f4cbe0, 0xc01a7f19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc056359dd0}, 0x660089, {0xc05a72ac60, {0x174301106bac1d71, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0182b0800, {0x63c85b8, 0xc056359dd0}, 0xc0551b3800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bac1d71, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bac1d71, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056359d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056359d40}, {0xc01a7f33f8, 0x660089}, 0xc01a7f3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056359d40}, {0x7fe394476e30}, 0xc0551b3600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056359cb0}, 0xc0551b3600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}, 0xc01696cd80, 0xc06ea00078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}, 0x3, 0xc00ff079c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a7f3980}}}, {0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}, 0xc01696cd80, 0xc00ff079c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056359cb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056359cb0}, {0x40, 0xc01a7f3b70}, 0xc01a7f3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056359cb0}, {0x4e9d7a0, 0xc0551b3600}, 0xc01696cd80, 0xc06ea00078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056359cb0}, 0xc04ebfb200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05a72aa20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05a72aa20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799766 [semacquire]: sync.runtime_SemacquireMutex(0x2568a42, 0x40, 0x7fe12b18c540) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc000088000, 0xc094ed2410, 0x49e6b9) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x119405e090166c7, 0x300, 0xc, 0x9baf000) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7236e27, 0x3b81}, 0x0, 0xc074c47d78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc074c47b00, {0xffffffffffffff15, 0xc094ed25b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc074c47b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc074c47b00, {0xc016048e80, 0x32, 0x7f}, 0x13, 0x15) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc074c47b00, {0xc03b455ce0, 0x32, 0x7f}, {0xc016048e80, 0x51c9860, 0xc016048e80}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc074c47b00, {0xc03b455ce0, 0x32, 0x70}, {0xc016048e80, 0x32, 0x7f}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0918dd998, {0xc03b455ce0, 0x32, 0x70}, {0xc016048e80, 0x32, 0x7f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0918dc2c0, {0xc016048e80, 0x7f, 0x70}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0918dc2c0, {0xc03b455ce0, 0x7f, 0xc016048e80}, {0xc016048e80, 0xc057650f30, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0918dc000, {0xc016048e80, 0x32, 0x7f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc03fdceda8, {{0xc057650f30, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04830ac00, {{0xc052a708d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc09e39b000, {0x63c85b8, 0xc04d1effb0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04d1effb0}, {0x64a6560, 0xc04830ac00}, {0xc052a708d0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04d1effb0}, {0x6466ed8, 0xc03fdceb00}, {0xc052a708d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04e22f1b8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05f741200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209bac00, {0x63c85b8, 0xc04d1effb0}, 0xc05f741200, 0xc062ad8140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209bac00, {0x63c85b8, 0xc04d1effb0}, 0xc05f741200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d1ef590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d1ef590}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d1ef590}, {0xc00102e4e0}, 0xc05f741100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d1ef530}, 0xc05f741100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01174b170}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810049 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bcfa000, {0x63c85b8, 0xc02954a600}, 0xc0b5d8a300, 0xc02eeaa1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bcfa000, {0x63c85b8, 0xc02954a600}, 0xc0b5d8a300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c0bde31, 0x0, 0x0}, 0xc044114cf0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c0bde31, 0x0, 0x0}, 0xc044114cf0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02954a570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02954a570}, {0xc00ecc13f8, 0x660089}, 0xc00ecc1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02954a570}, {0x7fe351f74048}, 0xc0b5d8a100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02954a4e0}, 0xc0b5d8a100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}, 0xc0656351a0, 0xc0269f4900) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}, 0x3, 0xc078e89380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ecc1980}}}, {0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}, 0xc0656351a0, 0xc078e89380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02954a4e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02954a4e0}, {0x40, 0xc00ecc1b70}, 0xc00ecc1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02954a4e0}, {0x4e9d7a0, 0xc0b5d8a100}, 0xc0656351a0, 0xc0269f4900) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02954a4e0}, 0xc052404de0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc073564d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc073564d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800859 [runnable]: sync.runtime_SemacquireMutex(0xc095efc690, 0x8f, 0xc095efc6a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc095efc538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x38, 0x2b, 0x38, 0xc095efc560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc014dfc480, {0xffffffffffffff15, 0xc095efc6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc014dfc480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc014dfc480, {0xc01316f0c0, 0x32, 0x34}, {0xc0311e9580, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc088262898, {0xc01316f0c0, 0x32, 0x34}, {0xc0311e9580, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0882611c0, {0xc0311e9580, 0x74, 0x34}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0882611c0, {0xc01316f0c0, 0x74, 0xc0311e9580}, {0xc0311e9580, 0xc04ca44510, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc088260f00, {0xc0311e9580, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc05af6c928, {{0xc04ca44510, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05c4b1b00, {{0xc066ca4c60, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04faff000, {0x63c85b8, 0xc01d6f2d80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01d6f2d80}, {0x64a6560, 0xc05c4b1b00}, {0xc066ca4c60, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01d6f2d80}, {0x6466ed8, 0xc05af6c680}, {0xc066ca4c60, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc065d80d98}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc028148800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c01800, {0x63c85b8, 0xc01d6f2d80}, 0xc028148800, 0xc068611900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c01800, {0x63c85b8, 0xc01d6f2d80}, 0xc028148800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01d6f2cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d6f2cf0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01d6f2cf0}, {0xc00102e4e0}, 0xc028148700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01d6f2c90}, 0xc028148700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a47e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812024 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc05f6fde90}, {0xc0e1b7e580, 0x4, 0xc0110c3e58}, {0x0, {0x1743010ff310f39f, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc05f6fde90}, 0x11310, 0xc03f5fb9e0, 0x20, 0xc0651624f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc05f6fde90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791404 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc04abe2ae0}, 0xc039a79cb0, {0x646cc48, 0xc04abe2b40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc04abe2ae0}, 0xedb7a834e, 0xc083c12480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc04abe2ae0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc056e94a80}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc039a79ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc056e94a80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802128 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc02ec1b980}, 0xc03e577ae8, 0x1, 0xc0596db0a8, 0x519bbc, 0xc038f389c0, 0xc0390ac060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02ec1b980}, 0x1e0, 0xc0596db230, 0x38f38960, 0x0, 0x660069, 0xc038f389c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc02ec1b980}, 0xc038f38960, {{{{0x0, 0x0}, {0xc04b897680, 0x4b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc032544120, {0x63c85b8, 0xc02ec1b980}, 0xc015d4d690, 0x596db428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc02ec1b980}, {0xc065d4a480, {0x174301101ea3318d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc02ec1b980}, 0xc051375900, 0xc0596db9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02ec1b980}, 0x660089, {0xc065d4a480, {0x174301101ea3318d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc02ec1b980}, 0xc0c7aa4a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ea3318d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ea3318d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ec1b8f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ec1b8f0}, {0xc0596dd3f8, 0x660089}, 0xc0596dd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ec1b8f0}, {0x7fe3c2f95230}, 0xc0c7aa4600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ec1b770}, 0xc0c7aa4600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}, 0xc01742b1c0, 0xc026f38fd8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}, 0x3, 0xc04f867500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0596dd980}}}, {0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}, 0xc01742b1c0, 0xc04f867500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02ec1b770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ec1b770}, {0x40, 0xc0596ddb70}, 0xc0596ddab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02ec1b770}, {0x4e9d7a0, 0xc0c7aa4600}, 0xc01742b1c0, 0xc026f38fd8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02ec1b770}, 0xc0266cc120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc079eb79e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc079eb79e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806591 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc09293cff0}, 0xc020748bd0, 0x1, 0x708017f130105d, 0x37e11d600, 0xc01bacf4a0, 0xc019c97800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc09293cff0}, 0x120, 0xc005051230, 0x1bacf440, 0x0, 0x660069, 0xc01bacf4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc09293cff0}, 0xc01bacf440, {{{{0x0, 0x0}, {0xc083859b00, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc09293cff0}, 0x6352410, 0x304bbcb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc09293cff0}, {0xc01bacf320, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc09293cff0}, 0xc04def9360, 0xc0050519f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc09293cff0}, 0x660089, {0xc01bacf320, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc09293cff0}, 0xc076736c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09293cf60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09293cf60}, {0xc0050533f8, 0x660089}, 0xc005053400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09293cf60}, {0x53454343410c1651}, 0xc076736a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09293ced0}, 0xc076736a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}, 0xc08e0fa920, 0xc020748ba0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}, 0x3, 0xc08fe0a940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005053980}}}, {0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}, 0xc08e0fa920, 0xc08fe0a940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09293ced0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09293ced0}, {0x40, 0xc005053b70}, 0xc005053ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09293ced0}, {0x4e9d7a0, 0xc076736a00}, 0xc08e0fa920, 0xc020748ba0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09293ced0}, 0xc03c46d080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01bacf200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc01bacf200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811363 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc048cddd70}, 0xc06292f638, 0x1, 0xc0111fb0a8, 0x519bbc, 0xc076d7a060, 0xc016c82f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc048cddd70}, 0x1e0, 0xc0111fb230, 0x76d7a000, 0x0, 0x660069, 0xc076d7a060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc048cddd70}, 0xc076d7a000, {{{{0x0, 0x0}, {0xc04ef61200, 0x116}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc048cddd70}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc048cddd70}, {0xc0308d7320, {0x17430110751e0f8c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc048cddd70}, 0xc0c36a97c0, 0xc0111fb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc048cddd70}, 0x660089, {0xc0308d7320, {0x17430110751e0f8c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc048cddd70}, 0xc045e95500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110751e0f8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110751e0f8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048cdd140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048cdd140}, {0xc0111fd3f8, 0x660089}, 0xc0111fd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048cdd140}, {0x7fe4286376e8}, 0xc045e95300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048cdcf30}, 0xc045e95300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}, 0xc0440b0360, 0xc06292f878) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}, 0x3, 0xc03763ba80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0111fd980}}}, {0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}, 0xc0440b0360, 0xc03763ba80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048cdcf30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048cdcf30}, {0x40, 0xc0111fdb70}, 0xc0111fdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048cdcf30}, {0x4e9d7a0, 0xc045e95300}, 0xc0440b0360, 0xc06292f878) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048cdcf30}, 0xc065e1d8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0308d7200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0308d7200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812221 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x1) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc082c01c80, 0xc092274f00, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x646b830, 0xc082c01c80, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x4d866e, 0x9185600}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).init(0xc0459d1b00, 0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1229 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2157 +0xdb github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a528c0, 0xc07cfc39a8, 0xc0922768e8, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc092276a00, 0x0, 0xc092276a00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc092276898, 0xc0734106e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092276898, {0xc023ed4440, 0x1f, 0x40}, {0xc08824a000, 0x1f, 0x1f}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0922751c0, {0xc08824a000, 0x4a2372, 0x1f}, 0x464ed60, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0922751c0, {0xc023ed4440, 0x1f, 0x1f}, {0xc08824a000, 0x1e, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092274f00, {0xc08824a000, 0x1f, 0x1f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc086b0cc28, {{0xc041011f80, 0x1e, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0820b5000, {0x63c85b8, 0xc045fa7170}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc045fa7170}, {0x64a6618, 0xc086b0cc28}, {0xc041011f80, 0x0, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc045fa7170}, {0x7fe429f83df0, 0xc049737740}, {0xc041011f80, 0x1e, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x63c85b8, 0xc045fa7170}, {0x7fe429f83df0, 0xc049737740}, {0xc041011f80, 0xc07cfc4818, 0x461247}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:617 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0, {0x63c85b8, 0xc045fa7170}, {0x7fe429f83df0, 0xc049737740}, {0x89, 0xed, 0xb4, 0x96, 0x59, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:98 +0xdb github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0x89, 0xed, 0xb4, 0x96, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:1958 +0xbc github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01f6a3f00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:201 +0x37e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a1a22455, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:644 +0x190 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a1a22455, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:612 +0x2da github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, {{0x17430110a1a22455, 0x0, 0x0}, {0x174301109e901fd8, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:428 +0x5f3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc017050000, {0x63c85b8, 0xc045fa7170}, {0xc03f16a010, 0x8}, 0xc01f6a3f00, {{0x17430110a1a22455, 0x0, 0x0}, {0x174301109e901fd8, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:673 +0x1c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x0, {0x63c85b8, _}, {_, _}, _, {{{0x1742da1aac0aa655, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:773 +0xaa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc017050000, {0x63c85b8, _}, _, _, {{{0x1742da1aac0aa655, 0x0, 0x0}, 0x0, {0x1f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:113 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc017050000, {0x63c85b8, 0xc045fa7170}, 0xc01f6a3f00, 0xc0ab9adf40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:167 +0x7b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017050000, {0x63c85b8, 0xc045fa7170}, 0xc01f6a3f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011083d4bf55, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011083d4bf55, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045fa70e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045fa70e0}, {0xc03e49b3f8, 0x660089}, 0xc03e49b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045fa70e0}, {0x31342d346230622d}, 0xc01f6a3900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045fa7050}, 0xc01f6a3900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}, 0xc0ccf7f0a0, 0xc038160c60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}, 0x3, 0xc0497376c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03e49b980}}}, {0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}, 0xc0ccf7f0a0, 0xc0497376c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045fa7050}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045fa7050}, {0x40, 0xc03e49bb70}, 0xc03e49bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045fa7050}, {0x4e9d7a0, 0xc01f6a3900}, 0xc0ccf7f0a0, 0xc038160c60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045fa7050}, 0xc037a09e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc050cdc6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc050cdc6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809363 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018aec640, {0x63c85b8, 0xc058fc0a80}, 0xc02a1b1740, 0x1, 0xc0210370a8, 0x519bbc, 0xc0679db260, 0xc0446b5020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc058fc0a80}, 0x120, 0xc021037230, 0x679db200, 0x0, 0x660069, 0xc0679db260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018aec640, {0x63c85b8, 0xc058fc0a80}, 0xc0679db200, {{{{0x0, 0x0}, {0xc03f321c20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc058fc0a80}, 0x6352410, 0x29b7af28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc008494e60, {0x63c85b8, 0xc058fc0a80}, {0xc0679db0e0, {0x1743011063e21f99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc008494e60, {0x63c85b8, 0xc058fc0a80}, 0xc01efdaa00, 0xc0210379f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc058fc0a80}, 0x660089, {0xc0679db0e0, {0x1743011063e21f99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000e4e800, {0x63c85b8, 0xc058fc0a80}, 0xc032a1fc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063e21f99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063e21f99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc058fc0930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058fc0930}, {0xc0210393f8, 0x660089}, 0xc021039400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc058fc0930}, {0x3fefffff78658355}, 0xc032a1fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc058fc0870}, 0xc032a1fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}, 0xc02fa35300, 0xc01e9be2b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}, 0x3, 0xc0525cd380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc021039980}}}, {0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}, 0xc02fa35300, 0xc0525cd380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc058fc0870}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058fc0870}, {0x40, 0xc021039b70}, 0xc021039ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc058fc0870}, {0x4e9d7a0, 0xc032a1fa00}, 0xc02fa35300, 0xc01e9be2b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc058fc0870}, 0xc01f5f60c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0679dafc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0679dafc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809406 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc0720a7da0}, 0xc01ce24588, 0x1, 0xc01a78b0a8, 0x519bbc, 0xc0888f1da0, 0xc060ca7c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0720a7da0}, 0x120, 0xc01a78b230, 0x888f1d40, 0x0, 0x660069, 0xc0888f1da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc0720a7da0}, 0xc0888f1d40, {{{{0x0, 0x0}, {0xc073d510e0, 0x1e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0720a7da0}, 0x6352410, 0x46e77988) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc0720a7da0}, {0xc0888f1c20, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc0720a7da0}, 0xc030b77180, 0xc01a78b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0720a7da0}, 0x660089, {0xc0888f1c20, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0720a7da0}, 0xc061064e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0720a7d10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0720a7d10}, {0xc01a78d3f8, 0x660089}, 0xc01a78d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0720a7d10}, {0x623437652d343462}, 0xc061064c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0720a7c80}, 0xc061064c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}, 0xc04cf66480, 0xc00beed188) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}, 0x3, 0xc058bed400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a78d980}}}, {0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}, 0xc04cf66480, 0xc058bed400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0720a7c80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0720a7c80}, {0x40, 0xc01a78db70}, 0xc01a78dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0720a7c80}, {0x4e9d7a0, 0xc061064c00}, 0xc04cf66480, 0xc00beed188) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0720a7c80}, 0xc074753ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086e53560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086e53560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812693 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219684913 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc048132000, {0x63c85b8, 0xc057ecc600}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc05a901900, {0x63c85b8, 0xc057ecc600}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc05a901900, {0x63c8510, 0xc0c1705e80}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc05a901900, {0x63c8510, 0xc0c1705e80}, {0x63f7db8, 0x9bf6f48}, 0xc08984b088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc05a901900, {0x63c85b8, 0xc04bce8c60}, {{0x63f7db8, 0x9bf6f48}, {0xc0a85af2a8, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc05a901900, {0x63c85b8, 0xc04bce8c60}, {{0x63f7db8, 0x9bf6f48}, {0xc0a85af2a8, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0a85af2a8, 0x6}, 0x0, 0x0}, {0x1d1eb2cd, 0xedb7a8350, 0x0}, {0x1d1eb686, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05a901900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05a901900, {0x63c8510, 0xc0131b0200}, 0xc088077dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0580d97e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0131b0200}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0580d97e0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219806564 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff688c0, {0x63c85b8, 0xc08d3c01b0}, 0xc08ecffcf8, 0x1, 0xc0029af0a8, 0x519bbc, 0xc088352600, 0xc05aa5f380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08d3c01b0}, 0x120, 0xc0029af230, 0x883525a0, 0x0, 0x660069, 0xc088352600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff688c0, {0x63c85b8, 0xc08d3c01b0}, 0xc0883525a0, {{{{0x0, 0x0}, {0xc050766a20, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08d3c01b0}, 0x6352410, 0x4207f580) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030006140, {0x63c85b8, 0xc08d3c01b0}, {0xc088352480, {0x174301104e71cee3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030006140, {0x63c85b8, 0xc08d3c01b0}, 0xc02aca4be0, 0xc0029af9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08d3c01b0}, 0x660089, {0xc088352480, {0x174301104e71cee3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75c00, {0x63c85b8, 0xc08d3c01b0}, 0xc066d2d300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e71cee3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e71cee3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08d3c0120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d3c0120}, {0xc0029b13f8, 0x660089}, 0xc0029b1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08d3c0120}, {0xc0029b1490}, 0xc066d2d000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08d3c0090}, 0xc066d2d000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}, 0xc08c18e1e0, 0xc025e37278) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}, 0x3, 0xc08a39c0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0029b1980}}}, {0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}, 0xc08c18e1e0, 0xc08a39c0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08d3c0090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08d3c0090}, {0x40, 0xc0029b1b70}, 0xc0029b1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08d3c0090}, {0x4e9d7a0, 0xc066d2d000}, 0xc08c18e1e0, 0xc025e37278) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08d3c0090}, 0xc048b41860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc063afad80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc063afad80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219748028 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab975e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc034e02780, 0xc01121d000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc034e02780, {0xc01121d000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc034e02780, {0xc01121d000, 0x735, 0xc04004cc80}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03894c878, {0xc01121d000, 0x7eda79, 0xc01121d005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0d4a0c900, {0xc01121d000, 0x5, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0d3544e58, {0xc01121d000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0643eb078, {0x631b900, 0xc0d3544e58}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0643eae00, {0x7fe42ab59310, 0xc0d4a0c900}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0643eae00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0643eae00, {0xc09d793000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc026e63290, {0xc09d793000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc09c0c9588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc09c0c9588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc09c0c9b60, {0x634aac0, 0xc09c0c9588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc09c0c9500, 0xc03f276540, 0xc07c59542b, 0xc07c59542d, {0x635bec0, 0xc061388360}, 0xc000d6ae00, 0xc024a180b8, 0xc07c595429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc09c0c9500, {0x63c85b8, 0xc0654c78f0}, 0xc03f276530, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc05eadf037, 0x8}}, 0x0, 0xc0654c7890, 0xc0654c78c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0654c78f0}, {0x643d810, 0xc0d4a0c900}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06ddb4c30}, {0x643d810, 0xc0d4a0c900}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06ddb4c30}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805399 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc078635140}, 0xc0525c1998, 0x1, 0xc03b2910a8, 0x519bbc, 0xc05ad5a720, 0xc05b7b9920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc078635140}, 0x120, 0xc03b291230, 0x5ad5a6c0, 0xc00b5429c0, 0x18, 0xc05ad5a720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc078635140}, 0xc05ad5a6c0, {{{{0x0, 0x0}, {0xc044135b00, 0x14}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc078635140}, 0x6352410, 0x3a07e028) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc078635140}, {0xc05ad5a5a0, {0x1743011041ec4a1c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc078635140}, 0xc05c449180, 0xc03b2919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc078635140}, 0x660089, {0xc05ad5a5a0, {0x1743011041ec4a1c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc078635140}, 0xc08b703700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041ec4a1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041ec4a1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0786350b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0786350b0}, {0xc03b2933f8, 0x660089}, 0xc03b293400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0786350b0}, {0xc0015221e0}, 0xc08b703500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078634fc0}, 0xc08b703500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}, 0xc0a0c54ba0, 0xc0525c1b30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}, 0x3, 0xc06b90afc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03b293980}}}, {0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}, 0xc0a0c54ba0, 0xc06b90afc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc078634fc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078634fc0}, {0x40, 0xc03b293b70}, 0xc03b293ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}, 0x20301a, 0x20301a) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc078634fc0}, {0x4e9d7a0, 0xc08b703500}, 0xc0a0c54ba0, 0xc0525c1b30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc078634fc0}, 0xc02fd399e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ad5a480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ad5a480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219796505 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0d50913e0}, 0xc0185cbcb0, {0x646cc48, 0xc0d5091440}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0d50913e0}, 0xedb7a834e, 0xc03799ea20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0d50913e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0277b2c90}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0185cbef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0277b2c90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219630110 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa00f8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0c5872880, 0xc043001800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0c5872880, {0xc043001800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0c5872880, {0xc043001800, 0x739, 0xc0330be3c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0463d6960, {0xc043001800, 0x7eda79, 0xc043001805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06cde0c00, {0xc043001800, 0xc0891f6c60, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc061e1a0f0, {0xc043001800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0cef7faf8, {0x631b900, 0xc061e1a0f0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0cef7f880, {0x7fe42ab59310, 0xc06cde0c00}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0cef7f880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0cef7f880, {0xc09772a000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc007701590, {0xc09772a000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc01a735588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc01a735588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc01a735b60, {0x634aac0, 0xc01a735588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc01a735500, 0xc060459b20, 0xc095adb42b, 0xc095adb42d, {0x635bec0, 0xc00c3b21b0}, 0xc000d6ae00, 0xc04c572ba8, 0xc095adb429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc01a735500, {0x63c85b8, 0xc029aa3440}, 0xc060459b10, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0941a804e, 0xb}}, 0x0, 0xc029aa33e0, 0xc029aa3410, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc029aa3440}, {0x643d810, 0xc06cde0c00}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc029e29140}, {0x643d810, 0xc06cde0c00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc029e29140}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807108 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc047e40090}, 0xc0304c6960, 0x1, 0xc00336f0a8, 0x519bbc, 0xc082cce300, 0xc06f706d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc08c012f00, {0x63c85b8, 0xc047e40090}, 0xc0379de120, 0xc00336f1a8, 0x82cce300, 0x0, 0x0, 0xc082cce300, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc047e40090}, 0xc04580e240, {{{{0xc09ed7e480, 0x36}, {0xc040250d80, 0x38}}, {{0xc044af3d40, 0x5f}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047e40090}, 0x63521a0, 0x321a1258) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc047e40090}, {0xc03c43b560, {0x1743011000664351, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc047e40090}, 0xc04c1c50e0, 0xc00336f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047e40090}, 0x660089, {0xc03c43b560, {0x1743011000664351, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc047e40090}, 0xc0895efa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011000664351, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011000664351, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e40000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e40000}, {0xc0033713f8, 0x660089}, 0xc003371400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e40000}, {0x3feffe1328e8ab1a}, 0xc0895ef700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc081bcbf20}, 0xc0895ef700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}, 0xc077d4c240, 0xc02620c9f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}, 0x3, 0xc0612c4440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003371980}}}, {0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}, 0xc077d4c240, 0xc0612c4440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc081bcbf20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc081bcbf20}, {0x40, 0xc003371b70}, 0xc003371ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc081bcbf20}, {0x4e9d7a0, 0xc0895ef700}, 0xc077d4c240, 0xc02620c9f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc081bcbf20}, 0xc04580e060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03c43b440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03c43b440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807526 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc0372e4180}, 0xc05897fcf8, 0x1, 0xc005ac30a8, 0x519bbc, 0xc08d13a060, 0xc0b9b52760) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0372e4180}, 0x120, 0xc005ac3230, 0x8d13a000, 0x0, 0x660069, 0xc08d13a060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc0372e4180}, 0xc08d13a000, {{{{0x0, 0x0}, {0xc073dae480, 0xf}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0372e4180}, 0x63526b0, 0x667624d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc0372e4180}, {0xc08b953e60, {0x1743011057c107bf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc0372e4180}, 0xc02fedef00, 0xc005ac39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0372e4180}, 0x660089, {0xc08b953e60, {0x1743011057c107bf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc0372e4180}, 0xc097ff0500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011057c107bf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011057c107bf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0372e40f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0372e40f0}, {0xc005ac53f8, 0x660089}, 0xc005ac5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0372e40f0}, {0x3e4b569d203e30ac}, 0xc097ff0300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0372e4060}, 0xc097ff0300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}, 0xc02e6be6a0, 0xc06c49b4b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}, 0x3, 0xc070c2a8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ac5980}}}, {0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}, 0xc02e6be6a0, 0xc070c2a8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0372e4060}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0372e4060}, {0x40, 0xc005ac5b70}, 0xc005ac5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0372e4060}, {0x4e9d7a0, 0xc097ff0300}, 0xc02e6be6a0, 0xc06c49b4b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0372e4060}, 0xc06c39cba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08b9537a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08b9537a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807057 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a63c0, {0x63c85b8, 0xc039393770}, 0xc04b8ecc30, 0x1, 0xc003df70a8, 0x519bbc, 0xc0826b33c0, 0xc05a07bbc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0629db560, {0x63c85b8, 0xc039393770}, 0xc0629db590, 0xc003df71a8, 0x41, 0x60, 0xffffffffffffffff, 0xc0826b33c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a63c0, {0x63c85b8, 0xc039393770}, 0xc0826b3180, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc05c2b4ea0, 0x2}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc039393770}, 0x63521a0, 0x1b928e58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddd10, {0x63c85b8, 0xc039393770}, {0xc07c140a20, {0x174301100370f598, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddd10, {0x63c85b8, 0xc039393770}, 0xc015ff5900, 0xc003df79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc039393770}, 0x660089, {0xc07c140a20, {0x174301100370f598, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e09b400, {0x63c85b8, 0xc039393770}, 0xc08d514700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100370f598, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100370f598, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0393936e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0393936e0}, {0xc003df93f8, 0x660089}, 0xc003df9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0393936e0}, {0x0}, 0xc08d514400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0393935f0}, 0xc08d514400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}, 0xc0402d59a0, 0xc02d7e2ed0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}, 0x3, 0xc0594ae200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003df9980}}}, {0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}, 0xc0402d59a0, 0xc0594ae200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0393935f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0393935f0}, {0x40, 0xc003df9b70}, 0xc003df9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0393935f0}, {0x4e9d7a0, 0xc08d514400}, 0xc0402d59a0, 0xc02d7e2ed0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0393935f0}, 0xc051a1c540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07c140900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07c140900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801981 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b7be800, {0x63c85b8, 0xc0454c7020}, 0xc071cd5600, 0xc03c3ef0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b7be800, {0x63c85b8, 0xc0454c7020}, 0xc071cd5600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101f20cece, 0x0, 0x0}, 0xc0535cbac0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101f20cece, 0x0, 0x0}, 0xc0535cbac0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0454c6f90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0454c6f90}, {0xc0536f93f8, 0x660089}, 0xc0536f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0454c6f90}, {0xc0015221e0}, 0xc071cd5400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0454c6f00}, 0xc071cd5400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}, 0xc01013b9e0, 0xc0382bbb00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}, 0x3, 0xc051c9f280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0536f9980}}}, {0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}, 0xc01013b9e0, 0xc051c9f280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0454c6f00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0454c6f00}, {0x40, 0xc0536f9b70}, 0xc0536f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0454c6f00}, {0x4e9d7a0, 0xc071cd5400}, 0xc01013b9e0, 0xc0382bbb00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0454c6f00}, 0xc039e5df20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03bcf0360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03bcf0360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812226 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc0111c6000}, 0xc077ba0c00, 0xc06ae134a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0111c6000}, 0xc077ba0c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e4955d4, 0x0, 0x0}, 0xc04f0256c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e4955d4, 0x0, 0x0}, 0xc04f0256c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02391be00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02391be00}, {0xc076115a08, 0x660089}, 0xc076115a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02391be00}, {0xc00102e4e0}, 0xc077ba0b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02391bda0}, 0xc077ba0b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc02391bce0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc02391bce0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802946 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc05b42f450, 0x5) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc040978000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc040978660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc039a0c000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc039a0c000, {0x63c8510, 0xc05b42f480}, 0xc07c441dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02ddbbe00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc05b42f480}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02ddbbe00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804600 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020f89900, {0x63c85b8, 0xc0607fe780}, 0xc0265000c0, 0x1, 0xc078d230a8, 0x519bbc, 0xc05310c600, 0xc040341c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0607fe780}, 0x1e0, 0xc078d23230, 0x5310c5a0, 0x0, 0x660069, 0xc05310c600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020f89900, {0x63c85b8, 0xc0607fe780}, 0xc05310c5a0, {{{{0x0, 0x0}, {0xc04fbe8d80, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0607fe780}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021106910, {0x63c85b8, 0xc0607fe780}, {0xc066b38120, {0x174301101d654fc6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021106910, {0x63c85b8, 0xc0607fe780}, 0xc046dcf040, 0xc078d239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0607fe780}, 0x660089, {0xc066b38120, {0x174301101d654fc6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021288000, {0x63c85b8, 0xc0607fe780}, 0xc05fa07f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d654fc6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d654fc6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0607fe6f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0607fe6f0}, {0xc078d253f8, 0x660089}, 0xc078d25400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0607fe6f0}, {0x7fe3b67188f8}, 0xc05fa07b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0607fe660}, 0xc05fa07b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}, 0xc0078193e0, 0xc04cebb260) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}, 0x3, 0xc056a8d500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078d25980}}}, {0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}, 0xc0078193e0, 0xc056a8d500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0607fe660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0607fe660}, {0x40, 0xc078d25b70}, 0xc078d25ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0607fe660}, {0x4e9d7a0, 0xc05fa07b00}, 0xc0078193e0, 0xc04cebb260) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0607fe660}, 0xc041ebb080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc066b38000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc066b38000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809020 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fd49cc0, {0x63c85b8, 0xc047147b00}, 0xc03d493950, 0x1, 0xc00fa590a8, 0x519bbc, 0xc0500f4d20, 0xc044be53e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01c38c7e0, {0x63c85b8, 0xc047147b00}, 0xc0500f4d20, 0xc00fa591a8, 0x31, 0x40, 0x0, 0xc0500f4d20, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fd49cc0, {0x63c85b8, 0xc047147b00}, 0xc0500f4c60, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc058adbe60, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047147b00}, 0x6352260, 0x290b5c58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fc6bcc0, {0x63c85b8, 0xc047147b00}, {0xc0500f4b40, {0x1743011025e86b69, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fc6bcc0, {0x63c85b8, 0xc047147b00}, 0xc02eeaba40, 0xc00fa599f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047147b00}, 0x660089, {0xc0500f4b40, {0x1743011025e86b69, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fda2000, {0x63c85b8, 0xc047147b00}, 0xc0b684b900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025e86b69, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025e86b69, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047147a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047147a40}, {0xc00fa5b3f8, 0x660089}, 0xc00fa5b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047147a40}, {0x0}, 0xc0b684b700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0471479b0}, 0xc0b684b700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}, 0xc044165000, 0xc03d493ae8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}, 0x3, 0xc05b727ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00fa5b980}}}, {0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}, 0xc044165000, 0xc05b727ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0471479b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0471479b0}, {0x40, 0xc00fa5bb70}, 0xc00fa5bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0471479b0}, {0x4e9d7a0, 0xc0b684b700}, 0xc044165000, 0xc03d493ae8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0471479b0}, 0xc026b2acc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0500f4900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0500f4900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810055 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0708c5320}, 0xc02eae3548, 0x1, 0xc01562d0a8, 0x519bbc, 0xc079ae0d20, 0xc041669620) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc08ac246c0, {0x63c85b8, 0xc0708c5320}, 0xc0792a2480, 0xc01562d1a8, 0x79ae0d20, 0x0, 0x2, 0xc079ae0d20, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0708c5320}, 0xc079ae0c60, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc03808bd40, 0x8e}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0708c5320}, 0x6352260, 0x3102f930) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0708c5320}, {0xc079ae0b40, {0x174301102e330700, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0708c5320}, 0xc068b2a6e0, 0xc01562d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0708c5320}, 0x660089, {0xc079ae0b40, {0x174301102e330700, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0708c5320}, 0xc063476f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e330700, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e330700, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0708c5230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0708c5230}, {0xc01562f3f8, 0x660089}, 0xc01562f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0708c5230}, {0x3fc3333333333333}, 0xc063476b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0708c5170}, 0xc063476b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}, 0xc0076a0c00, 0xc0d1f76900) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}, 0x3, 0xc078c20080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01562f980}}}, {0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}, 0xc0076a0c00, 0xc078c20080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0708c5170}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0708c5170}, {0x40, 0xc01562fb70}, 0xc01562fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0708c5170}, {0x4e9d7a0, 0xc063476b00}, 0xc0076a0c00, 0xc0d1f76900) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0708c5170}, 0xc020132600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc079ae0900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc079ae0900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806266 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0871a30b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805249 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f7a5b80, {0x63c85b8, 0xc0473cf0e0}, 0xc04d66bae8, 0x1, 0xc0789a10a8, 0x519bbc, 0xc05a8eb5c0, 0xc05a49d6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0473cf0e0}, 0x120, 0xc0789a1230, 0x5a8eb560, 0xc05bc94000, 0x18, 0xc05a8eb5c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f7a5b80, {0x63c85b8, 0xc0473cf0e0}, 0xc05a8eb560, {{{{0x0, 0x0}, {0xc0573687e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0473cf0e0}, 0x6352410, 0x1cdd8550) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f663720, {0x63c85b8, 0xc0473cf0e0}, {0xc05a8eb440, {0x174301103f2af520, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f663720, {0x63c85b8, 0xc0473cf0e0}, 0xc00ab57ea0, 0xc0789a19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0473cf0e0}, 0x660089, {0xc05a8eb440, {0x174301103f2af520, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f7e7400, {0x63c85b8, 0xc0473cf0e0}, 0xc091102e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f2af520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f2af520, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0473cf050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473cf050}, {0xc0789a33f8, 0x660089}, 0xc0789a3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0473cf050}, {0x4d939c}, 0xc091102c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0473cef60}, 0xc091102c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}, 0xc0654e3a60, 0xc04d91e8a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}, 0x3, 0xc061075780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0789a3980}}}, {0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}, 0xc0654e3a60, 0xc061075780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0473cef60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473cef60}, {0x40, 0xc0789a3b70}, 0xc0789a3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}, 0x203018, 0x203018) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0473cef60}, {0x4e9d7a0, 0xc091102c00}, 0xc0654e3a60, 0xc04d91e8a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0473cef60}, 0xc0baffee40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a8eb320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a8eb320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805958 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc067bb3dd0}, 0xc031856f78, 0x1, 0xc00580bff0, 0x519bbc, 0xc05ed75aa0, 0xc05fe08d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc067bb3dd0}, 0x1e0, 0xc00580c178, 0x5ed75a40, 0x0, 0x660069, 0xc05ed75aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc067bb3dd0}, 0xc05ed75a40, {{{{0x0, 0x0}, {0xc059a34b40, 0x5a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc067bb3dd0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc067bb3dd0}, {0xc06bd42b40, {0x174301104a2798a9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc067bb3dd0}, 0xc055b23a40, 0xc057c74938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc067bb3dd0}, 0x660089, {0xc06bd42b40, {0x174301104a2798a9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc067bb3dd0}, 0xc04cfc8400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104a2798a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104a2798a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067bb3d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067bb3d40}, {0xc057c76340, 0x660089}, 0xc057c76348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067bb3d40}, {0xc00102e4e0}, 0xc04cfc8300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067bb3ce0}, 0xc04cfc8300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc026af7a10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812728 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc0532faa80, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc040b8b198, 0xc0931a2938, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc0931a2a50, 0x0, 0xc0931a2a50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0931a28e8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0931a28e8, {0xc0a91c8910, 0x18, 0x50}, {0xc0430c9420, 0x18, 0x6f}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0931a16c0, {0xc0430c9420, 0x0, 0xc040b8b4e0}, 0x121ea30, 0x10) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0931a16c0, {0xc0a91c8910, 0x18, 0x6f}, {0xc0430c9420, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0931a1400, {0xc0430c9420, 0x18, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05a08c588, {{0xc08e777398, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04eeb8600, {{0xc08e777398, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f585400, {0x63c85b8, 0xc062cdec00}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062cdec00}, {0x64a6560, 0xc04eeb8600}, {0xc08e777398, 0x636166756e616d6e, 0x18}, {0x1743011095a80acd, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062cdec00}, {0x6466ed8, 0xc05a08c580}, {0xc08e777398, 0x17, 0x18}, {0x1743011095a80acd, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04318f878}, {{0x1743011095a80acd, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011095a80acd, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc050ef3000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b3756fcd, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d6400, {0x63c85b8, 0xc062cdec00}, 0xc050ef3000, 0xc020c63860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d6400, {0x63c85b8, 0xc062cdec00}, 0xc050ef3000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011095a80acd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011095a80acd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cdeb70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdeb70}, {0xc040b913f8, 0x660089}, 0xc040b91400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cdeb70}, {0x5245545349474552}, 0xc050ef2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cdeae0}, 0xc050ef2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}, 0xc06ac12bc0, 0xc049096150) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}, 0x3, 0xc01c0bde40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc040b91980}}}, {0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}, 0xc06ac12bc0, 0xc01c0bde40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cdeae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdeae0}, {0x40, 0xc040b91b70}, 0xc040b91ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cdeae0}, {0x4e9d7a0, 0xc050ef2e00}, 0xc06ac12bc0, 0xc049096150) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cdeae0}, 0xc0492578c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f8000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f8000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812766 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812579 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812764 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791429 [IO wait]: internal/poll.runtime_pollWait(0x7fe456a4e3a8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc068856980, 0xc04c1bb800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc068856980, {0xc04c1bb800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc068856980, {0xc04c1bb800, 0x735, 0xc0c376c140}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc051b89e68, {0xc04c1bb800, 0x7eda79, 0xc04c1bb805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc020e0b500, {0xc04c1bb800, 0x3, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03c545c38, {0xc04c1bb800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc063b63078, {0x631b900, 0xc03c545c38}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc063b62e00, {0x7fe42ab59310, 0xc020e0b500}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc063b62e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc063b62e00, {0xc0dd91e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04c7460d8, {0xc0dd91e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0b279b588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0b279b588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0b279bb60, {0x634aac0, 0xc0b279b588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0b279b500, 0xc00df8ab90, 0xc077d6b42b, 0xc077d6b42d, {0x635bec0, 0xc0827cafc0}, 0xc000d6ae00, 0xc011294a48, 0xc077d6b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0b279b500, {0x63c85b8, 0xc077e70c60}, 0xc00df8ab80, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08dad803e, 0x8}}, 0x0, 0xc077e70c00, 0xc077e70c30, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc077e70c60}, {0x643d810, 0xc020e0b500}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03c51bcb0}, {0x643d810, 0xc020e0b500}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03c51bcb0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811641 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc001606dc0, {0x63c85b8, 0xc0580a4d80}, 0xc0195a7308, 0x1, 0xc025be50a8, 0x519bbc, 0xc065ff3380, 0xc04a44af00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0580a4d80}, 0x120, 0xc025be5230, 0x65ff3320, 0x0, 0x660069, 0xc065ff3380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc001606dc0, {0x63c85b8, 0xc0580a4d80}, 0xc065ff3320, {{{{0x0, 0x0}, {0xc063208d80, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0580a4d80}, 0x6352410, 0x3824cdd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00a993450, {0x63c85b8, 0xc0580a4d80}, {0xc065ff3200, {0x174301106f6cccce, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00a993450, {0x63c85b8, 0xc0580a4d80}, 0xc07d8bed20, 0xc025be59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0580a4d80}, 0x660089, {0xc065ff3200, {0x174301106f6cccce, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001acf800, {0x63c85b8, 0xc0580a4d80}, 0xc06fd2eb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f6cccce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f6cccce, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0580a4cc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0580a4cc0}, {0xc025be73f8, 0x660089}, 0xc025be7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0580a4cc0}, {0x7fe3e0f56238}, 0xc06fd2e900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0580a4c30}, 0xc06fd2e900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}, 0xc06b015240, 0xc03ed20d20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}, 0x3, 0xc01637ea80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc025be7980}}}, {0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}, 0xc06b015240, 0xc01637ea80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0580a4c30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0580a4c30}, {0x40, 0xc025be7b70}, 0xc025be7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0580a4c30}, {0x4e9d7a0, 0xc06fd2e900}, 0xc06b015240, 0xc03ed20d20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0580a4c30}, 0xc03690d860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc065ff30e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc065ff30e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809991 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc0594df500}, 0xc0411c81c8, 0x1, 0x70801805d973a8, 0x37e11d600, 0xc023226720, 0xc07d459800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0594df500}, 0x120, 0xc02805b230, 0x232266c0, 0x0, 0x660069, 0xc023226720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc0594df500}, 0xc0232266c0, {{{{0x0, 0x0}, {0xc03caef9e0, 0x1f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0594df500}, 0x6352410, 0x29b7bfc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc0594df500}, {0xc023226240, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc0594df500}, 0xc049eddd60, 0xc02805b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0594df500}, 0x660089, {0xc023226240, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc0594df500}, 0xc084dc6300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0594df410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0594df410}, {0xc02805d3f8, 0x660089}, 0xc02805d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0594df410}, {0x0}, 0xc084dc6000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0594df320}, 0xc084dc6000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}, 0xc032ba51c0, 0xc0411c8198) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}, 0x3, 0xc05b30ce80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02805d980}}}, {0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}, 0xc032ba51c0, 0xc05b30ce80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0594df320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0594df320}, {0x40, 0xc02805db70}, 0xc02805dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0594df320}, {0x4e9d7a0, 0xc084dc6000}, 0xc032ba51c0, 0xc0411c8198) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0594df320}, 0xc020577c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c5e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c5e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219658806 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af29e88, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc097561380, 0xc0497be800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc097561380, {0xc0497be800, 0x745, 0x745}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc097561380, {0xc0497be800, 0x745, 0xc05539a780}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc05a5cf688, {0xc0497be800, 0x7eda79, 0xc0497be805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02b77af00, {0xc0497be800, 0xc085b68c60, 0x745}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02902d110, {0xc0497be800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc078ea0cf8, {0x631b900, 0xc02902d110}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc078ea0a80, {0x7fe42ab59310, 0xc02b77af00}, 0x745) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc078ea0a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc078ea0a80, {0xc0a2622000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0289a9b90, {0xc0a2622000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05f686088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05f686088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05f686660, {0x634aac0, 0xc05f686088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05f686000, 0xc04189de90, 0xc089a6942b, 0xc089a6942d, {0x635bec0, 0xc0336c9440}, 0xc000d6ae00, 0xc040127070, 0xc089a69429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05f686000, {0x63c85b8, 0xc051450f60}, 0xc04189de80, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc089363009, 0xd}}, 0x0, 0xc051450ed0, 0xc051450f00, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc051450f60}, {0x643d810, 0xc02b77af00}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc081dba870}, {0x643d810, 0xc02b77af00}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc081dba870}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804706 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f65b000, {0x63c85b8, 0xc04024eed0}, 0xc04d14a900, 0xc0178df9a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f65b000, {0x63c85b8, 0xc04024eed0}, 0xc04d14a900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffb90bf72, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffb90bf72, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04024ee40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04024ee40}, {0xc0787513f8, 0x660089}, 0xc078751400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04024ee40}, {0xc0015221e0}, 0xc04d14a700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04024edb0}, 0xc04d14a700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}, 0xc0994b5a40, 0xc032264030) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}, 0x3, 0xc036dcc880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078751980}}}, {0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}, 0xc0994b5a40, 0xc036dcc880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04024edb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04024edb0}, {0x40, 0xc078751b70}, 0xc078751ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04024edb0}, {0x4e9d7a0, 0xc04d14a700}, 0xc0994b5a40, 0xc032264030) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04024edb0}, 0xc027fdacc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc060cf8ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc060cf8ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806194 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc04ea679e0}, 0xc092e19800, 0xc0b02820a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04ea679e0}, 0xc092e19800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104bee1d80, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104bee1d80, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ea67950}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ea67950}, {0xc006a6d3f8, 0x660089}, 0xc006a6d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ea67950}, {0xd0300a280a200118}, 0xc092e19500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ea678c0}, 0xc092e19500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}, 0xc01e7a4920, 0xc08bf9cf00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}, 0x3, 0xc0418938c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006a6d980}}}, {0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}, 0xc01e7a4920, 0xc0418938c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ea678c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ea678c0}, {0x40, 0xc006a6db70}, 0xc006a6dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ea678c0}, {0x4e9d7a0, 0xc092e19500}, 0xc01e7a4920, 0xc08bf9cf00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ea678c0}, 0xc0616db620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc040eb9680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc040eb9680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806415 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc07bf2d4d0}, 0xc038d5a600, 0x1, 0x0, 0x1, 0xc07d089aa0, 0xc0729a27e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07bf2d4d0}, 0x1e0, 0xc0042f6728, 0x7d089a40, 0x0, 0x660069, 0xc07d089aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc07bf2d4d0}, 0xc07d089a40, {{{{0x0, 0x0}, {0xc058a9a480, 0x5e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc03f853860, {0x63c85b8, 0xc07bf2d4d0}, 0xc00451bd70, 0x7c43e920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc07bf2d4d0}, {0xc072298b40, {0x174301104dbcbff6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc07bf2d4d0}, 0xc049e9b900, 0xc07c43eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07bf2d4d0}, 0x660089, {0xc072298b40, {0x174301104dbcbff6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07bf2d4d0}, 0xc09b7d8f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104dbcbff6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104dbcbff6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07bf2d440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bf2d440}, {0xc0d68368f0, 0x660089}, 0xc0d68368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07bf2d440}, {0xc00102e4e0}, 0xc09b7d8e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07bf2d3e0}, 0xc09b7d8e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07c7c3b30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801188 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc094aa8548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc094aa85b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0914091f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc058f40e00, {0xc05fc70c30, 0x2b, 0x2f}, {0xc022372d80, 0x2b, 0x76}, 0x1, 0x91) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091409088, {0xc05fc70c30, 0x1, 0x76}, {0xc022372d80, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091409088, {0xc05fc70c30, 0x2b, 0x2f}, {0xc022372d80, 0x2b, 0x76}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0914087c0, {0xc022372d80, 0x76, 0x2f}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0914087c0, {0xc05fc70c30, 0x2b, 0xc022372d80}, {0xc022372d80, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091408500, {0xc022372d80, 0x2b, 0x76}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05c467088, {{0xc03e8771d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0629fad00, {{0xc03e8771d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ea45400, {0x63c85b8, 0xc01e58a300}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01e58a300}, {0x64a6560, 0xc0629fad00}, {0xc03e8771d0, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01e58a300}, {0x6466ed8, 0xc05c467080}, {0xc03e8771d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc049dadbd8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0545f7200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c23c00, {0x63c85b8, 0xc01e58a300}, 0xc0545f7200, 0xc0ab4d6be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c23c00, {0x63c85b8, 0xc01e58a300}, 0xc0545f7200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e58a270}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e58a270}, {0xc09363a8f0, 0x660089}, 0xc09363a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e58a270}, {0xc00102e4e0}, 0xc0545f6f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e58a210}, 0xc0545f6f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc076faf9e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807627 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc082fa31d0}, 0xc01de860c0, 0x1, 0xc0073ef0a8, 0x519bbc, 0xc03ff1ba40, 0xc0974475c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc082fa31d0}, 0x120, 0xc0073ef230, 0x3ff1b9e0, 0x0, 0x660069, 0xc03ff1ba40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc082fa31d0}, 0xc03ff1b9e0, {{{{0x0, 0x0}, {0xc056fd2d80, 0x22}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc082fa31d0}, 0x6352410, 0x53415a18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc082fa31d0}, {0xc03ff1b8c0, {0x1743011058bf3457, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc082fa31d0}, 0xc030c65220, 0xc0073ef9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc082fa31d0}, 0x660089, {0xc03ff1b8c0, {0x1743011058bf3457, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc082fa31d0}, 0xc04aea3d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058bf3457, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058bf3457, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc082fa30e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082fa30e0}, {0xc0073f13f8, 0x660089}, 0xc0073f1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc082fa30e0}, {0x0}, 0xc04aea3200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc082fa3050}, 0xc04aea3200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}, 0xc086665e40, 0xc03bb184c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}, 0x3, 0xc0141e4a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0073f1980}}}, {0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}, 0xc086665e40, 0xc0141e4a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc082fa3050}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082fa3050}, {0x40, 0xc0073f1b70}, 0xc0073f1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc082fa3050}, {0x4e9d7a0, 0xc04aea3200}, 0xc086665e40, 0xc03bb184c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc082fa3050}, 0xc04e118e40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03ff1b7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03ff1b7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812408 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff688c0, {0x63c85b8, 0xc04f6fcb10}, 0xc03c10ce40, 0x1, 0xc03f0130a8, 0x519bbc, 0xc05396d920, 0xc062bf95c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04f6fcb10}, 0x120, 0xc03f013230, 0x5396d8c0, 0x0, 0x660069, 0xc05396d920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff688c0, {0x63c85b8, 0xc04f6fcb10}, 0xc05396d8c0, {{{{0x0, 0x0}, {0xc0434a7560, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f6fcb10}, 0x6352410, 0x17839218) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030006140, {0x63c85b8, 0xc04f6fcb10}, {0xc05396d7a0, {0x174301107ca83f7d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030006140, {0x63c85b8, 0xc04f6fcb10}, 0xc03884ebe0, 0xc03f0139f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f6fcb10}, 0x660089, {0xc05396d7a0, {0x174301107ca83f7d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75c00, {0x63c85b8, 0xc04f6fcb10}, 0xc079621a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107ca83f7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107ca83f7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f6fca80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f6fca80}, {0xc03f0153f8, 0x660089}, 0xc03f015400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f6fca80}, {0x656422202c393032}, 0xc079621800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f6fc9f0}, 0xc079621800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}, 0xc087886700, 0xc043a91d40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}, 0x3, 0xc058616a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03f015980}}}, {0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}, 0xc087886700, 0xc058616a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f6fc9f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f6fc9f0}, {0x40, 0xc03f015b70}, 0xc03f015ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f6fc9f0}, {0x4e9d7a0, 0xc079621800}, 0xc087886700, 0xc043a91d40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f6fc9f0}, 0xc04d5f79e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05396cb40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05396cb40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812752 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804651 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc023b90b40, {0x63c85b8, 0xc047e9acc0}, 0xc0299e2618, 0x1, 0x708017d5ad5ca5, 0x37e11d600, 0xc0682943c0, 0xc03c6394a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc047e9acc0}, 0x120, 0xc090541230, 0x68294360, 0xc0b8501520, 0x18, 0xc0682943c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc023b90b40, {0x63c85b8, 0xc047e9acc0}, 0xc068294360, {{{{0x0, 0x0}, {0xc0468c4b40, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc047e9acc0}, 0x6352410, 0x421b1638) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc023a88ff0, {0x63c85b8, 0xc047e9acc0}, {0xc068294240, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc023a88ff0, {0x63c85b8, 0xc047e9acc0}, 0xc02e5fac80, 0xc0905419f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc047e9acc0}, 0x660089, {0xc068294240, {0x1743011024922b76, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023ba7c00, {0x63c85b8, 0xc047e9acc0}, 0xc08dbb2c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024922b76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e9ac30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e9ac30}, {0xc0905433f8, 0x660089}, 0xc090543400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e9ac30}, {0x7fe34e276a30}, 0xc08dbb2a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e9aba0}, 0xc08dbb2a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}, 0xc013bea300, 0xc0299e25e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}, 0x3, 0xc03e23ee80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc090543980}}}, {0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}, 0xc013bea300, 0xc03e23ee80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc047e9aba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e9aba0}, {0x40, 0xc090543b70}, 0xc090543ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}, 0x20300f, 0x20300f) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc047e9aba0}, {0x4e9d7a0, 0xc08dbb2a00}, 0xc013bea300, 0xc0299e25e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc047e9aba0}, 0xc01c47eba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06ab797a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06ab797a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812594 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799515 [runnable]: sync.runtime_SemacquireMutex(0xc0872cc690, 0x8f, 0x7fe3a20ecd40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0872cc538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0872cc560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc045d66900, {0xffffffffffffff15, 0xc0872cc6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc045d66900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc045d66900, {0xc01dbbc980, 0x32, 0x77}, {0xc02f41a000, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b1487998, {0xc01dbbc980, 0x32, 0x77}, {0xc02f41a000, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b14862c0, {0xc02f41a000, 0x74, 0x77}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b14862c0, {0xc01dbbc980, 0x74, 0xc02f41a000}, {0xc02f41a000, 0xc070435ef0, 0x31}, 0x85) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b1486000, {0xc02f41a000, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc05af6a828, {{0xc070435ef0, 0x31, 0x85}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0375eea00, {{0xc023631560, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08f3f6800, {0x63c85b8, 0xc04fa32600}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04fa32600}, {0x64a6560, 0xc0375eea00}, {0xc023631560, 0x21036, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04fa32600}, {0x6466ed8, 0xc05af6a580}, {0xc023631560, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07d48fdd0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03287d400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0205cb800, {0x63c85b8, 0xc04fa32600}, 0xc03287d400, 0xc07a98a5a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cb800, {0x63c85b8, 0xc04fa32600}, 0xc03287d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04fa32570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04fa32570}, {0xc0036f68f0, 0x660089}, 0xc0036f68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04fa32570}, {0xc00102e4e0}, 0xc03287d300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04fa32510}, 0xc03287d300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc79380}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801301 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02532b400, {0x63c85b8, 0xc086a788d0}, 0xc0622fcb00, 0xc02ad1f860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc086a788d0}, 0xc0622fcb00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ae0ee73, 0x0, 0x0}, 0xc03dc417a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ae0ee73, 0x0, 0x0}, 0xc03dc417a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086a78840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086a78840}, {0xc052ba73f8, 0x660089}, 0xc052ba7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086a78840}, {0x3033324341129678}, 0xc0622fc900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086a787b0}, 0xc0622fc900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}, 0xc02782b480, 0xc020122420) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}, 0x3, 0xc03bd81ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc052ba7980}}}, {0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}, 0xc02782b480, 0xc03bd81ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086a787b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086a787b0}, {0x40, 0xc052ba7b70}, 0xc052ba7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086a787b0}, {0x4e9d7a0, 0xc0622fc900}, 0xc02782b480, 0xc020122420) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086a787b0}, 0xc07942bc80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc077120c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc077120c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806441 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc088a40150}, 0xc024690480, 0x1, 0x0, 0x1, 0xc02cd7f920, 0xc0752cd6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc088a40150}, 0x120, 0xc00b40a728, 0x2cd7f8c0, 0x0, 0x660069, 0xc02cd7f920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc088a40150}, 0xc02cd7f8c0, {{{{0x0, 0x0}, {0xc09ecf18c0, 0x106}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc088a40150}, 0x6352410, 0x321a0738) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc088a40150}, {0xc02cd7f7a0, {0x174301104eac03bc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc088a40150}, 0xc08768adc0, 0xc075e56ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc088a40150}, 0x660089, {0xc02cd7f7a0, {0x174301104eac03bc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc088a40150}, 0xc05c5ed300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104eac03bc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104eac03bc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc088a400c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc088a400c0}, {0xc0036aa8f0, 0x660089}, 0xc0036aa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc088a400c0}, {0xc00102e4e0}, 0xc05c5ed200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc088a40060}, 0xc05c5ed200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc083279f50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802174 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc05ca98dc0, {0x63c85b8, 0xc024fd1ec0}, 0xc0558d06d8, 0x1, 0xc0590390a8, 0x519bbc, 0xc03da814a0, 0xc057a02720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc024fd1ec0}, 0x120, 0xc059039230, 0x3da81440, 0x0, 0x660069, 0xc03da814a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc05ca98dc0, {0x63c85b8, 0xc024fd1ec0}, 0xc03da81440, {{{{0x0, 0x0}, {0xc0baba5320, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc024fd1ec0}, 0x6352410, 0x41bc4048) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc045008410, {0x63c85b8, 0xc024fd1ec0}, {0xc03da81320, {0x1743011020584514, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc045008410, {0x63c85b8, 0xc024fd1ec0}, 0xc02f38f040, 0xc0590399f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024fd1ec0}, 0x660089, {0xc03da81320, {0x1743011020584514, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc06ffeec00, {0x63c85b8, 0xc024fd1ec0}, 0xc0ab105000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020584514, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020584514, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024fd1c20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024fd1c20}, {0xc05903b3f8, 0x660089}, 0xc05903b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024fd1c20}, {0xc0015221e0}, 0xc0ab104e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024fd1aa0}, 0xc0ab104e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}, 0xc00a33ce80, 0xc0a29e7140) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}, 0x3, 0xc03ccf4280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05903b980}}}, {0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}, 0xc00a33ce80, 0xc03ccf4280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc024fd1aa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024fd1aa0}, {0x40, 0xc05903bb70}, 0xc05903bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc024fd1aa0}, {0x4e9d7a0, 0xc0ab104e00}, 0xc00a33ce80, 0xc0a29e7140) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc024fd1aa0}, 0xc02785a540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03da81200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03da81200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812405 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch.func4.1({0x63c8580, 0xc00be47b00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:301 +0xf6 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc075ea2180}, {0x4f0948f, 0x11}, 0x4a817c800, 0xc07e11cef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch.func4({0x63c85b8, 0xc075ea2180}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:299 +0x91 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790042 [runnable]: internal/poll.runtime_pollWait(0x7fe42ab98030, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc058461180, 0xc0531db800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc058461180, {0xc0531db800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc058461180, {0xc0531db800, 0x735, 0xc0cd53a280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc029b7b5b0, {0xc0531db800, 0x7eda79, 0xc0531db805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0b16fb560, {0xc0531db800, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0bba63a28, {0xc0531db800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0526753f8, {0x631b900, 0xc0bba63a28}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc052675180, {0x7fe42ab59310, 0xc0b16fb560}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc052675180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc052675180, {0xc0b4c36000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06fab0630, {0xc0b4c36000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc068502b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc068502b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0685030e0, {0x634aac0, 0xc068502b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc068502a80, 0xc028d40700, 0xc09678d42b, 0xc09678d42d, {0x635bec0, 0xc03722ef30}, 0xc000d6ae00, 0xc015c68b08, 0xc09678d429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc068502a80, {0x63c85b8, 0xc01b6fac30}, 0xc028d406f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc095a7e038, 0x8}}, 0x0, 0xc01b6fabd0, 0xc01b6fac00, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc01b6fac30}, {0x643d810, 0xc0b16fb560}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02514cdb0}, {0x643d810, 0xc0b16fb560}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02514cdb0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802153 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc064b07920}, 0xc047405710, 0x1, 0xc01a4550a8, 0x519bbc, 0xc09efbfa40, 0xc000f0e2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc064b07920}, 0x120, 0xc01a455230, 0x9efbf9e0, 0x0, 0x660069, 0xc09efbfa40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc064b07920}, 0xc09efbf9e0, {{{{0x0, 0x0}, {0xc0639cfc20, 0x10}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc064b07920}, 0x6352410, 0x351a9970) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc064b07920}, {0xc09efbf8c0, {0x1743010fec5acf9a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc064b07920}, 0xc06f1a8320, 0xc01a4559f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc064b07920}, 0x660089, {0xc09efbf8c0, {0x1743010fec5acf9a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc064b07920}, 0xc06d00f400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fec5acf9a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fec5acf9a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064b07890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064b07890}, {0xc01a4573f8, 0x660089}, 0xc01a457400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064b07890}, {0xc0015221e0}, 0xc06d00f200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064b07800}, 0xc06d00f200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}, 0xc04ecdafa0, 0xc0302f5c68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}, 0x3, 0xc06e1c71c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a457980}}}, {0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}, 0xc04ecdafa0, 0xc06e1c71c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc064b07800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064b07800}, {0x40, 0xc01a457b70}, 0xc01a457ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc064b07800}, {0x4e9d7a0, 0xc06d00f200}, 0xc04ecdafa0, 0xc0302f5c68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc064b07800}, 0xc04ad09260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0221b4b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0221b4b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806876 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0929ab860}, 0xc04e68b410, 0x1, 0x708017f03d7bde, 0x37e11d600, 0xc065c035c0, 0xc03a488180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0929ab860}, 0x120, 0xc00433b230, 0x65c03560, 0x0, 0x660069, 0xc065c035c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0929ab860}, 0xc065c03560, {{{{0x0, 0x0}, {0xc082ccd200, 0x60}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0929ab860}, 0x6352410, 0x1b928a98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0929ab860}, {0xc065c03440, {0x1743011045df03b1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0929ab860}, 0xc0ce753720, 0xc00433b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0929ab860}, 0x660089, {0xc065c03440, {0x1743011045df03b1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0929ab860}, 0xc07a042200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011045df03b1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011045df03b1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0929ab7d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0929ab7d0}, {0xc00433d3f8, 0x660089}, 0xc00433d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0929ab7d0}, {0x3f99abec8835f379}, 0xc07a042000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0929ab740}, 0xc07a042000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}, 0xc0907f94e0, 0xc04e68b3c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}, 0x3, 0xc08e0d1e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00433d980}}}, {0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}, 0xc0907f94e0, 0xc08e0d1e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0929ab740}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0929ab740}, {0x40, 0xc00433db70}, 0xc00433dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0929ab740}, {0x4e9d7a0, 0xc07a042000}, 0xc0907f94e0, 0xc04e68b3c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0929ab740}, 0xc0310a96e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc065c03320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc065c03320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805593 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc066ad9860}, 0xc06f2650c8, 0x1, 0x708017e13704ad, 0x37e11d600, 0xc05ef2b4a0, 0xc05b77eba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc066ad9860}, 0x120, 0xc00296b230, 0x5ef2b440, 0xc0a3567a00, 0x18, 0xc05ef2b4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc066ad9860}, 0xc05ef2b440, {{{{0x0, 0x0}, {0xc05b77ab40, 0x25}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc066ad9860}, 0x6352410, 0x17bb3b08) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc066ad9860}, {0xc05ef2b320, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc066ad9860}, 0xc04cf16b40, 0xc00296b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066ad9860}, 0x660089, {0xc05ef2b320, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc066ad9860}, 0xc0472aa700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066ad97d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066ad97d0}, {0xc00296d3f8, 0x660089}, 0xc00296d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066ad97d0}, {0x7fe3f72dbb98}, 0xc0472aa500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066ad9740}, 0xc0472aa500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}, 0xc06f236860, 0xc06f265098) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}, 0x3, 0xc077be39c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00296d980}}}, {0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}, 0xc06f236860, 0xc077be39c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066ad9740}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066ad9740}, {0x40, 0xc00296db70}, 0xc00296dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}, 0x20301d, 0x20301d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066ad9740}, {0x4e9d7a0, 0xc0472aa500}, 0xc06f236860, 0xc06f265098) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066ad9740}, 0xc03c26e420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ef2b200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ef2b200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809864 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04d0a4fc0}, 0xc066ab5830, 0x1, 0xc00ae9b0a8, 0x519bbc, 0xc02c761bc0, 0xc03690ec00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05f94b200, {0x63c85b8, 0xc04d0a4fc0}, 0xc0a0991680, 0xc00ae9b1a8, 0x2c761bc0, 0x0, 0x0, 0xc02c761bc0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04d0a4fc0}, 0xc02c761b00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0b08bbd40, 0x89}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d0a4fc0}, 0x6352260, 0x3102f378) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04d0a4fc0}, {0xc02c761680, {0x174301102bed681e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04d0a4fc0}, 0xc05b44e460, 0xc00ae9b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d0a4fc0}, 0x660089, {0xc02c761680, {0x174301102bed681e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04d0a4fc0}, 0xc05b35fd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bed681e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bed681e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d0a4f30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d0a4f30}, {0xc00ae9d3f8, 0x660089}, 0xc00ae9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d0a4f30}, {0x3fefffdc5f52ef1d}, 0xc05b35fb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d0a4ea0}, 0xc05b35fb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}, 0xc0631cf1a0, 0xc0456aeab0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}, 0x3, 0xc0151b7b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ae9d980}}}, {0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}, 0xc0631cf1a0, 0xc0151b7b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d0a4ea0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d0a4ea0}, {0x40, 0xc00ae9db70}, 0xc00ae9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d0a4ea0}, {0x4e9d7a0, 0xc05b35fb00}, 0xc0631cf1a0, 0xc0456aeab0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d0a4ea0}, 0xc06e3bbc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc02c760900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc02c760900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812777 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810597 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024887180, {0x63c85b8, 0xc046f454a0}, 0xc0bb3a85b8, 0x1, 0xc02d0f30a8, 0x519bbc, 0xc014d975c0, 0xc0843d64e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046f454a0}, 0x120, 0xc02d0f3230, 0x14d97560, 0x0, 0x660069, 0xc014d975c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024887180, {0x63c85b8, 0xc046f454a0}, 0xc014d97560, {{{{0x0, 0x0}, {0xc059e47560, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046f454a0}, 0x6352410, 0x55bf2150) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0249460f0, {0x63c85b8, 0xc046f454a0}, {0xc014d97320, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0249460f0, {0x63c85b8, 0xc046f454a0}, 0xc074116be0, 0xc02d0f39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046f454a0}, 0x660089, {0xc014d97320, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024954c00, {0x63c85b8, 0xc046f454a0}, 0xc086e92000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046f45410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046f45410}, {0xc02d0f53f8, 0x660089}, 0xc02d0f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046f45410}, {0x6178706e69736878}, 0xc0d1d3fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046f45350}, 0xc0d1d3fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}, 0xc041cabb40, 0xc052163608) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}, 0x3, 0xc0a7c90cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02d0f5980}}}, {0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}, 0xc041cabb40, 0xc0a7c90cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046f45350}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046f45350}, {0x40, 0xc02d0f5b70}, 0xc02d0f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046f45350}, {0x4e9d7a0, 0xc0d1d3fd00}, 0xc041cabb40, 0xc052163608) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046f45350}, 0xc02bce6f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0457b4a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0457b4a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810259 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02c4a8c00, {0x63c85b8, 0xc07a57e3c0}, 0xc01e10f000, 0xc055056be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc07a57e3c0}, 0xc01e10f000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c0d638a, 0x0, 0x0}, 0xc03a65c2c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c0d638a, 0x0, 0x0}, 0xc03a65c2c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a57e330}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a57e330}, {0xc01aafb3f8, 0x660089}, 0xc01aafb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a57e330}, {0xc0015221e0}, 0xc01e10ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a57e2a0}, 0xc01e10ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}, 0xc00fb8e2e0, 0xc02532d458) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}, 0x3, 0xc088761400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aafb980}}}, {0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}, 0xc00fb8e2e0, 0xc088761400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a57e2a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a57e2a0}, {0x40, 0xc01aafbb70}, 0xc01aafbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a57e2a0}, {0x4e9d7a0, 0xc01e10ee00}, 0xc00fb8e2e0, 0xc02532d458) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a57e2a0}, 0xc0b96422a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc073564fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc073564fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812552 [runnable]: sync.runtime_SemacquireMutex(0xc041a18fa8, 0x6c, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x8, 0xc00cade480) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01e9c1080, {0x804f921, 0x466a}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc00cade480, {0xffffffffffffff15, 0xc041a191d8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc00cade480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc00cade480, {0xc0368126e0, 0x4b, 0x97}, {0xc061624700, 0x4b, 0x6d}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0894a2d98, {0xc0368126e0, 0x4b, 0x97}, {0xc061624700, 0x4b, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0894a16c0, {0xc061624700, 0x0, 0xc041a194e0}, 0x121ea30, 0xb0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0894a16c0, {0xc0368126e0, 0x4b, 0x6d}, {0xc061624700, 0x4a, 0x50}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0894a1400, {0xc061624700, 0x4b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc061c26588, {{0xc02c1bbae0, 0x4a, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07c29b500, {{0xc02c1bbae0, 0x4a, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07dbeac00, {0x63c85b8, 0xc056614750}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056614750}, {0x64a6560, 0xc07c29b500}, {0xc02c1bbae0, 0x6e6f69737265765f, 0x50}, {0x174301108d890fb5, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056614750}, {0x6466ed8, 0xc061c26580}, {0xc02c1bbae0, 0x4a, 0x50}, {0x174301108d890fb5, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04e22ed98}, {{0x174301108d890fb5, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301108d890fb5, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc083617600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110ab5674b5, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc000e4e800, {0x63c85b8, 0xc056614750}, 0xc083617600, 0xc076db0be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000e4e800, {0x63c85b8, 0xc056614750}, 0xc083617600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108d890fb5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108d890fb5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0566146c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0566146c0}, {0xc041a1f3f8, 0x660089}, 0xc041a1f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0566146c0}, {0x6f305978716f582b}, 0xc083617400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056614630}, 0xc083617400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}, 0xc0ccf7fc20, 0xc038161c98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}, 0x3, 0xc071e54b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc041a1f980}}}, {0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}, 0xc0ccf7fc20, 0xc071e54b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056614630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614630}, {0x40, 0xc041a1fb70}, 0xc041a1fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056614630}, {0x4e9d7a0, 0xc083617400}, 0xc0ccf7fc20, 0xc038161c98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056614630}, 0xc071982840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809370 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02f7c3000, {0x63c85b8, 0xc04558fad0}, 0xc04830a800, 0xc050934640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f7c3000, {0x63c85b8, 0xc04558fad0}, 0xc04830a800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105af607c5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105af607c5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04558fa40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04558fa40}, {0xc0168553f8, 0x660089}, 0xc016855400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04558fa40}, {0x7fe3921e0dc8}, 0xc04830a300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04558f9b0}, 0xc04830a300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}, 0xc03329ea00, 0xc01cb98288) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}, 0x3, 0xc06fb54000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc016855980}}}, {0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}, 0xc03329ea00, 0xc06fb54000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04558f9b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04558f9b0}, {0x40, 0xc016855b70}, 0xc016855ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04558f9b0}, {0x4e9d7a0, 0xc04830a300}, 0xc03329ea00, 0xc01cb98288) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04558f9b0}, 0xc04efe3f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06ca80120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06ca80120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812100 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f5377c0, {0x63c85b8, 0xc03feb3110}, 0xc0ce7a47f8, 0x1, 0xc032e3d0a8, 0x519bbc, 0xc06bac24e0, 0xc04b16bc00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03feb3110}, 0x120, 0xc032e3d230, 0x6bac2480, 0x0, 0x660069, 0xc06bac24e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f5377c0, {0x63c85b8, 0xc03feb3110}, 0xc06bac2480, {{{{0x0, 0x0}, {0xc07afbfe60, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03feb3110}, 0x63526b0, 0x655a21b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f552640, {0x63c85b8, 0xc03feb3110}, {0xc06bac2360, {0x174301107d70939f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f552640, {0x63c85b8, 0xc03feb3110}, 0xc06d7fedc0, 0xc032e3d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03feb3110}, 0x660089, {0xc06bac2360, {0x174301107d70939f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ffa0400, {0x63c85b8, 0xc03feb3110}, 0xc0641ff300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107d70939f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107d70939f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03feb3080}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03feb3080}, {0xc032e3f3f8, 0x660089}, 0xc032e3f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03feb3080}, {0x7fe3f1e6f170}, 0xc0641ff100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03feb2ff0}, 0xc0641ff100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}, 0xc02aa67940, 0xc0ce7a5098) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}, 0x3, 0xc030bb9ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032e3f980}}}, {0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}, 0xc02aa67940, 0xc030bb9ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03feb2ff0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03feb2ff0}, {0x40, 0xc032e3fb70}, 0xc032e3fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03feb2ff0}, {0x4e9d7a0, 0xc0641ff100}, 0xc02aa67940, 0xc0ce7a5098) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03feb2ff0}, 0xc089acbc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06bac2000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06bac2000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802925 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e096000, {0x63c85b8, 0xc020155b00}, 0xc065904b00, 0xc04858adc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e096000, {0x63c85b8, 0xc020155b00}, 0xc065904b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102802a3d8, 0x0, 0x0}, 0xc065530de0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102802a3d8, 0x0, 0x0}, 0xc065530de0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020155a10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020155a10}, {0xc06318b918, 0x660089}, 0xc06318b920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020155a10}, {0xc00102e4e0}, 0xc065904a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0201559b0}, 0xc065904a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc0413fc840}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc020155920}, {0xc00dfd36b0, 0x24}, 0x483115252, 0xc0385e1d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc03eaaea20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc020155920}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809173 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc03de0aea0}, 0xc006594cb0, {0x646cc48, 0xc03de0af00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc03de0aea0}, 0xedb7a8350, 0xc00872be60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc03de0aea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc07bc0f050}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc006594ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc07bc0f050}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807347 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc03c63e0f0}, 0xc0387e77a0, 0x1, 0xc0036570a8, 0x519bbc, 0xc09245ca00, 0xc03a90a420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc08efa3500, {0x63c85b8, 0xc03c63e0f0}, 0xc050565c20, 0xc0036571a8, 0x4bd14e, 0x48d7e7, 0xc003657130, 0xc09245ca00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc03c63e0f0}, 0xc04d3e8a20, {{{{0xc09ed7e480, 0x36}, {0xc04f93fd40, 0x3e}}, {{0xc0480d1200, 0x61}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03c63e0f0}, 0x63521a0, 0x4f4554b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc03c63e0f0}, {0xc07a701b00, {0x1743010ff082d0d1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc03c63e0f0}, 0xc0496fc3c0, 0xc0036579f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03c63e0f0}, 0x660089, {0xc07a701b00, {0x1743010ff082d0d1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc03c63e0f0}, 0xc08e597900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff082d0d1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff082d0d1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c63e060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c63e060}, {0xc0036593f8, 0x660089}, 0xc003659400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c63e060}, {0x3feffe1328e8ab1a}, 0xc08e597600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc025813e00}, 0xc08e597600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}, 0xc06c8c6c80, 0xc050c587f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}, 0x3, 0xc0a3efc900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003659980}}}, {0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}, 0xc06c8c6c80, 0xc0a3efc900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc025813e00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc025813e00}, {0x40, 0xc003659b70}, 0xc003659ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc025813e00}, {0x4e9d7a0, 0xc08e597600}, 0xc06c8c6c80, 0xc050c587f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc025813e00}, 0xc04d3e88a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07a7019e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07a7019e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805777 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc0860281b0}, 0xc09c8d4c00, 0xc055f2a640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0860281b0}, 0xc09c8d4c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c6e8840, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c6e8840, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086028120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086028120}, {0xc006fd53f8, 0x660089}, 0xc006fd5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086028120}, {0xc0015221e0}, 0xc09c8d4a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086028090}, 0xc09c8d4a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}, 0xc029f23980, 0xc08bb53d40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}, 0x3, 0xc0265f7e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006fd5980}}}, {0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}, 0xc029f23980, 0xc0265f7e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086028090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086028090}, {0x40, 0xc006fd5b70}, 0xc006fd5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086028090}, {0x4e9d7a0, 0xc09c8d4a00}, 0xc029f23980, 0xc08bb53d40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086028090}, 0xc0465e6e40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0450519e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0450519e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802058 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc0a67e89f0}, 0xc027e30d98, 0x1, 0xc05495f0a8, 0x519bbc, 0xc0175a4960, 0xc05af72ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0a67e89f0}, 0x120, 0xc05495f230, 0x175a4900, 0x0, 0x660069, 0xc0175a4960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc0a67e89f0}, 0xc0175a4900, {{{{0x0, 0x0}, {0xc055d199e0, 0x13}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0a67e89f0}, 0x6352410, 0x23fbac90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc0a67e89f0}, {0xc0175a4480, {0x1743010fec5acf9a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc0a67e89f0}, 0xc049d0c960, 0xc05495f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0a67e89f0}, 0x660089, {0xc0175a4480, {0x1743010fec5acf9a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc0a67e89f0}, 0xc041ab3000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fec5acf9a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fec5acf9a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0a67e8960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a67e8960}, {0xc0549613f8, 0x660089}, 0xc054961400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0a67e8960}, {0x7fe3b8b26060}, 0xc041ab2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0a67e88d0}, 0xc041ab2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}, 0xc04db78780, 0xc03bdd5bc0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}, 0x3, 0xc02443bd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc054961980}}}, {0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}, 0xc04db78780, 0xc02443bd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0a67e88d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a67e88d0}, {0x40, 0xc054961b70}, 0xc054961ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0a67e88d0}, {0x4e9d7a0, 0xc041ab2e00}, 0xc04db78780, 0xc03bdd5bc0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0a67e88d0}, 0xc02f533ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0175a4000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0175a4000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810175 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6000, {0x63c85b8, 0xc03e3716b0}, 0xc06f29d9b0, 0x1, 0xc0246f10a8, 0x519bbc, 0xc057879e00, 0xc060b3f1a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc060493b00, {0x63c85b8, 0xc03e3716b0}, 0xc060493b48, 0xc0246f11a8, 0x41, 0x60, 0xffffffffffffffff, 0xc057879e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6000, {0x63c85b8, 0xc03e3716b0}, 0xc057879d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc042e5ca20, 0xe}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03e3716b0}, 0x6352260, 0x7b2287d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddb30, {0x63c85b8, 0xc03e3716b0}, {0xc057879c20, {0x17430110306734ae, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc03e3716b0}, 0xc051625680, 0xc0246f19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03e3716b0}, 0x660089, {0xc057879c20, {0x17430110306734ae, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc03e3716b0}, 0xc061e91600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03e3715f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03e3715f0}, {0xc0246f33f8, 0x660089}, 0xc0246f3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03e3715f0}, {0x7fdd4a3590f8}, 0xc061e91400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03e371500}, 0xc061e91400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}, 0xc02906a920, 0xc04ebf7f68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}, 0x3, 0xc05a45e640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0246f3980}}}, {0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}, 0xc02906a920, 0xc05a45e640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03e371500}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03e371500}, {0x40, 0xc0246f3b70}, 0xc0246f3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03e371500}, {0x4e9d7a0, 0xc061e91400}, 0xc02906a920, 0xc04ebf7f68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03e371500}, 0xc0be9dbaa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057879b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057879b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219369614 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc089582210, {0x63c85b8, 0xc02fdb6c90}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc00f8b0000, {0x63c85b8, 0xc02fdb6c90}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc00f8b0000, {0x63c85b8, 0xc02fdb67b0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc00f8b0000, {0x63c85b8, 0xc02fdb67b0}, {0x63f7db8, 0x9bf6f48}, 0xc0652a3088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc00f8b0000, {0x63c85b8, 0xc06c854ba0}, {{0x63f7db8, 0x9bf6f48}, {0xc0167043ea, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc00f8b0000, {0x63c85b8, 0xc06c854ba0}, {{0x63f7db8, 0x9bf6f48}, {0xc0167043ea, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0167043ea, 0x6}, 0x0, 0x0}, {0xcd143b6, 0xedb7a8350, 0x0}, {0xcd14956, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc00f8b0000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc00f8b0000, {0x63c8510, 0xc0b617e080}, 0xc07d823dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03e54dfa0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0b617e080}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03e54dfa0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811689 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc06199c6f0}, 0xc013d6cab0, 0x1, 0x70801819100800, 0x37e11d600, 0xc065f5c840, 0xc07e33e840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06199c6f0}, 0x120, 0xc0054c3230, 0x65f5c7e0, 0x0, 0x660069, 0xc065f5c840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc06199c6f0}, 0xc065f5c7e0, {{{{0x0, 0x0}, {0xc0457ba6c0, 0x33}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06199c6f0}, 0x6352410, 0x33a923a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc06199c6f0}, {0xc065f5c6c0, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc06199c6f0}, 0xc06cd2fea0, 0xc0054c39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06199c6f0}, 0x660089, {0xc065f5c6c0, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc06199c6f0}, 0xc060cc9a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06199c660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06199c660}, {0xc0054c53f8, 0x660089}, 0xc0054c5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06199c660}, {0x7fe3ba6d0190}, 0xc060cc9800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06199c5d0}, 0xc060cc9800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}, 0xc0262c6680, 0xc013d6ca80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}, 0x3, 0xc04e09e5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0054c5980}}}, {0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}, 0xc0262c6680, 0xc04e09e5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06199c5d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06199c5d0}, {0x40, 0xc0054c5b70}, 0xc0054c5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06199c5d0}, {0x4e9d7a0, 0xc060cc9800}, 0xc0262c6680, 0xc013d6ca80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06199c5d0}, 0xc063da73e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc007904fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc007904fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800251 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc037add2f0}, 0xc035b7bc50, 0x1, 0xc040ce70a8, 0x519bbc, 0xc0250fbaa0, 0xc0273a9c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc037add2f0}, 0x1e0, 0xc040ce7230, 0x250fba40, 0x0, 0x660069, 0xc0250fbaa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc037add2f0}, 0xc0250fba40, {{{{0x0, 0x0}, {0xc04d0f2900, 0x33}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc037add2f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc037add2f0}, {0xc06d8f0a20, {0x1743011012ffa0c9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc037add2f0}, 0xc04eeab540, 0xc040ce79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc037add2f0}, 0x660089, {0xc06d8f0a20, {0x1743011012ffa0c9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc037add2f0}, 0xc09d4b6400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011012ffa0c9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011012ffa0c9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc037add260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037add260}, {0xc040ce93f8, 0x660089}, 0xc040ce9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc037add260}, {0x0}, 0xc09d4b6200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc037add1a0}, 0xc09d4b6200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}, 0xc0839879a0, 0xc023c7b248) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}, 0x3, 0xc066d34480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc040ce9980}}}, {0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}, 0xc0839879a0, 0xc066d34480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc037add1a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc037add1a0}, {0x40, 0xc040ce9b70}, 0xc040ce9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc037add1a0}, {0x4e9d7a0, 0xc09d4b6200}, 0xc0839879a0, 0xc023c7b248) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc037add1a0}, 0xc0438f0060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06d8f0900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06d8f0900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800547 [semacquire]: sync.runtime_SemacquireMutex(0xc00074a760, 0x0, 0x11840a5) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).Lock(0xc09523c338) GOROOT/src/sync/rwmutex.go:116 +0x71 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fe19e464f40, 0x7e32, 0x7fe3f6c18bc0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc020554c00, 0x7fe19e464f01, 0x7fe19e464f60, 0x3fb8, 0x3fbd) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc011f08840, {0x1f76a8b, 0x3fb8}, 0x0, 0xc091cfcff8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc091cfcd80, {0xffffffffffffff15, 0xc09523c5c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc091cfcd80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc091cfcd80, {0xc038fdb030, 0x2b, 0x6d}, 0x13, 0x1d) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc091cfcd80, {0xc02a675480, 0x2b, 0x6d}, {0xc038fdb030, 0x51c9860, 0xc038fdb030}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc091cfcd80, {0xc02a675480, 0x2b, 0x73}, {0xc038fdb030, 0x2b, 0x6d}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091741998, {0xc02a675480, 0x2b, 0x73}, {0xc038fdb030, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0917402c0, {0xc038fdb030, 0x6d, 0x73}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0917402c0, {0xc02a675480, 0x2b, 0xc038fdb030}, {0xc038fdb030, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091740000, {0xc038fdb030, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc09c449608, {{0xc0198038c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc058b30600, {{0xc0198038c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc08d570400, {0x63c85b8, 0xc01a4f3350}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01a4f3350}, {0x64a6560, 0xc058b30600}, {0xc0198038c0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01a4f3350}, {0x6466ed8, 0xc09c449600}, {0xc0198038c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07c51a960}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04a0d1000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020aeec00, {0x63c85b8, 0xc01a4f3350}, 0xc04a0d1000, 0xc072a890e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020aeec00, {0x63c85b8, 0xc01a4f3350}, 0xc04a0d1000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01a4f32c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01a4f32c0}, {0xc07d2528f0, 0x660089}, 0xc07d2528f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01a4f32c0}, {0xc00102e4e0}, 0xc04a0d0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01a4f3260}, 0xc04a0d0f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7ba10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219796359 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff35940, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05961a480, 0xc00b4e0800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05961a480, {0xc00b4e0800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05961a480, {0xc00b4e0800, 0x739, 0xc054588b40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc025d615c0, {0xc00b4e0800, 0x7eda79, 0xc00b4e0805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0a8297f20, {0xc00b4e0800, 0xc09c416c60, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0450fd6e0, {0xc00b4e0800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc081ffb778, {0x631b900, 0xc0450fd6e0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc081ffb500, {0x7fe42ab59310, 0xc0a8297f20}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc081ffb500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc081ffb500, {0xc0a42e5000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06089f530, {0xc0a42e5000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05c712b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05c712b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05c7130e0, {0x634aac0, 0xc05c712b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05c712a80, 0xc08aba2ba0, 0xc09c41742b, 0xc09c41742d, {0x635bec0, 0xc06d4f61b0}, 0xc000d6ae00, 0xc04e338dc4, 0xc09c417429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05c712a80, {0x63c85b8, 0xc025fabfb0}, 0xc08aba2b90, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc061b18021, 0xb}}, 0x0, 0xc025fabf50, 0xc025fabf80, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc025fabfb0}, {0x643d810, 0xc0a8297f20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc017376de0}, {0x643d810, 0xc0a8297f20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc017376de0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801019 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0969c8548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0969c85b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc092d840f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0d134e700, {0xc02354cc00, 0x2b, 0x72}, {0xc04db2c910, 0x2b, 0x4a}, 0x1, 0x40) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092d83f88, {0xc02354cc00, 0x1, 0x4a}, {0xc04db2c910, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092d83f88, {0xc02354cc00, 0x2b, 0x72}, {0xc04db2c910, 0x2b, 0x4a}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092d836c0, {0xc04db2c910, 0x4a, 0x72}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092d836c0, {0xc02354cc00, 0x2b, 0xc04db2c910}, {0xc04db2c910, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092d83400, {0xc04db2c910, 0x2b, 0x4a}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06c036c08, {{0xc02971f110, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0bc3cba00, {{0xc02971f110, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0bbeaf800, {0x63c85b8, 0xc04094ea20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04094ea20}, {0x64a6560, 0xc0bc3cba00}, {0xc02971f110, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04094ea20}, {0x6466ed8, 0xc06c036c00}, {0xc02971f110, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc035a185e8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05365d400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c22400, {0x63c85b8, 0xc04094ea20}, 0xc05365d400, 0xc047208e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c22400, {0x63c85b8, 0xc04094ea20}, 0xc05365d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04094e930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04094e930}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04094e930}, {0xc00102e4e0}, 0xc05365d300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04094e8d0}, 0xc05365d300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a4e40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808221 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc0769d8ed0}, 0xc036e9fef0, 0x1, 0xc006fe10a8, 0x519bbc, 0xc07cc2dec0, 0xc0648a42a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0769d8ed0}, 0x120, 0xc006fe1230, 0x7cc2de60, 0x0, 0x660069, 0xc07cc2dec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc0769d8ed0}, 0xc07cc2de60, {{{{0x0, 0x0}, {0xc064246b40, 0x2b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0769d8ed0}, 0x6352410, 0x2ad6fc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc0769d8ed0}, {0xc07cc2dd40, {0x174301105b266044, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc0769d8ed0}, 0xc058e0d220, 0xc006fe19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0769d8ed0}, 0x660089, {0xc07cc2dd40, {0x174301105b266044, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc0769d8ed0}, 0xc04ed2fb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b266044, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b266044, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0769d8e40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0769d8e40}, {0xc006fe33f8, 0x660089}, 0xc006fe3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0769d8e40}, {0x7fe3c1562d88}, 0xc04ed2f900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0769d8db0}, 0xc04ed2f900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}, 0xc09d3b7bc0, 0xc01c027248) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}, 0x3, 0xc064cc0c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006fe3980}}}, {0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}, 0xc09d3b7bc0, 0xc064cc0c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0769d8db0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0769d8db0}, {0x40, 0xc006fe3b70}, 0xc006fe3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0769d8db0}, {0x4e9d7a0, 0xc04ed2f900}, 0xc09d3b7bc0, 0xc01c027248) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0769d8db0}, 0xc075b39aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07cc2dc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07cc2dc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811245 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0210c1c00, {0x63c85b8, 0xc005c3eb70}, 0xc08d1b4b00, 0xc072e68a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c1c00, {0x63c85b8, 0xc005c3eb70}, 0xc08d1b4b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110743b5d1d, 0x0, 0x0}, 0xc03cd41620, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110743b5d1d, 0x0, 0x0}, 0xc03cd41620, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc005c3eae0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc005c3eae0}, {0xc028d4f3f8, 0x660089}, 0xc028d4f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc005c3eae0}, {0xc0015221e0}, 0xc08d1b4900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc005c3ea50}, 0xc08d1b4900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}, 0xc081b09d80, 0xc025b97b48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}, 0x3, 0xc09ac19740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028d4f980}}}, {0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}, 0xc081b09d80, 0xc09ac19740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc005c3ea50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc005c3ea50}, {0x40, 0xc028d4fb70}, 0xc028d4fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc005c3ea50}, {0x4e9d7a0, 0xc08d1b4900}, 0xc081b09d80, 0xc025b97b48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc005c3ea50}, 0xc050204660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06eb95b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06eb95b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801441 [runnable]: sync.runtime_SemacquireMutex(0xc09802e690, 0x8f, 0xc09802e478) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x39c3, 0x0, 0x1194e0b) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x2cb00000013, 0x3c, 0x3c, 0xc09802e560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc00bd94d80, {0xffffffffffffff15, 0xc09802e6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc00bd94d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc00bd94d80, {0xc0347048c0, 0x32, 0x63}, {0xc031aecf80, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092e96d98, {0xc0347048c0, 0x32, 0x63}, {0xc031aecf80, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092e956c0, {0xc031aecf80, 0x77, 0x63}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092e956c0, {0xc0347048c0, 0x77, 0xc031aecf80}, {0xc031aecf80, 0xc02a972140, 0x31}, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092e95400, {0xc031aecf80, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0390258a8, {{0xc02a972140, 0x31, 0x40}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0a0753600, {{0xc047ee6e10, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04e5e5400, {0x63c85b8, 0xc07028fe30}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc07028fe30}, {0x64a6560, 0xc0a0753600}, {0xc047ee6e10, 0x494745422d2d4017, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc07028fe30}, {0x6466ed8, 0xc039025600}, {0xc047ee6e10, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9ab70}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0a1a6f500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020d8ac00, {0x63c85b8, 0xc07028fe30}, 0xc0a1a6f500, 0xc05b011720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020d8ac00, {0x63c85b8, 0xc07028fe30}, 0xc0a1a6f500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07028fda0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07028fda0}, {0xc0032fc8f0, 0x660089}, 0xc0032fc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07028fda0}, {0xc00102e4e0}, 0xc0a1a6f400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07028fd40}, 0xc0a1a6f400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02ef02a80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219696747 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0b9e80210, {0x63c85b8, 0xc0403aaf90}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc054cae600, {0x63c85b8, 0xc0403aaf90}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc054cae600, {0x63c8510, 0xc05636d980}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc054cae600, {0x63c8510, 0xc05636d980}, 0xc04f7973a0, {0x7fe42b790cf0, 0xc040f326c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc054cae600, {0x63c85b8, 0xc077694cc0}, {{0x63f88f8, 0xc04f7973a0}, {0xc01fbbcaaf, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc054cae600, {0x63c85b8, 0xc077694cc0}, {{0x63f88f8, 0xc04f7973a0}, {0xc01fbbcaaf, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc04f7973a0}, {0xc01fbbcaaf, 0x23}, 0x0, 0x0}, {0xe486bbc, 0xedb7a8350, 0x0}, {0xe48729b, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc054cae600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc054cae600, {0x63c8510, 0xc0cd3a6a80}, 0xc07b513dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc027041490) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0cd3a6a80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc027041490) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219803244 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc0573b6300}, 0xc05a3899c8, 0x1, 0xc05725d0a8, 0x519bbc, 0xc036842a80, 0xc0366b9da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0573b6300}, 0x120, 0xc05725d230, 0x36842a20, 0x0, 0x660069, 0xc036842a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc0573b6300}, 0xc036842a20, {{{{0x0, 0x0}, {0xc0592eaa20, 0x19}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0573b6300}, 0x6352410, 0x14c75070) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc0573b6300}, {0xc036842900, {0x174301102bd75419, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc0573b6300}, 0xc028113400, 0xc05725d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0573b6300}, 0x660089, {0xc036842900, {0x174301102bd75419, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0573b6300}, 0xc05a615b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bd75419, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bd75419, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0573b6270}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0573b6270}, {0xc05725f3f8, 0x660089}, 0xc05725f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0573b6270}, {0xc0015221e0}, 0xc05a615900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0573b61e0}, 0xc05a615900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}, 0xc08bd6b780, 0xc0542d7620) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}, 0x3, 0xc023e29a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05725f980}}}, {0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}, 0xc08bd6b780, 0xc023e29a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0573b61e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0573b61e0}, {0x40, 0xc05725fb70}, 0xc05725fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0573b61e0}, {0x4e9d7a0, 0xc05a615900}, 0xc08bd6b780, 0xc0542d7620) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0573b61e0}, 0xc0385ab6e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0368427e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0368427e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809809 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc078773c80}, 0xc044c12c30, 0x1, 0xc001ddd0a8, 0x519bbc, 0xc06e413740, 0xc07801e1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc03c43ad80, {0x63c85b8, 0xc078773c80}, 0xc03c43ae50, 0xc001ddd1a8, 0x26, 0x40, 0xffffffffffffffff, 0xc06e413740, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc078773c80}, 0xc06e413500, {{{{0xc05fad0c60, 0x10}, {0xc09cbdd440, 0x21}}, {{0xc088046b40, 0x40}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc078773c80}, 0x63521a0, 0x4b1bd640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc078773c80}, {0xc066c1ab40, {0x174301100efa2c20, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc078773c80}, 0xc07ad9c500, 0xc001ddd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc078773c80}, 0x660089, {0xc066c1ab40, {0x174301100efa2c20, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc078773c80}, 0xc048877700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100efa2c20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100efa2c20, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078773bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078773bf0}, {0xc001ddf3f8, 0x660089}, 0xc001ddf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078773bf0}, {0x0}, 0xc048877500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078773b00}, 0xc048877500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}, 0xc078f0e120, 0xc072e1e6f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}, 0x3, 0xc0293ce9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc001ddf980}}}, {0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}, 0xc078f0e120, 0xc0293ce9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc078773b00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078773b00}, {0x40, 0xc001ddfb70}, 0xc001ddfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc078773b00}, {0x4e9d7a0, 0xc048877500}, 0xc078f0e120, 0xc072e1e6f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc078773b00}, 0xc00e02d6e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04aa29320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04aa29320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812553 [sleep]: time.Sleep(0x3b9aca00) GOROOT/src/runtime/time.go:193 +0x12e github.com/cockroachdb/cockroach/pkg/server.(*statusServer).CancelQueryByKey.func1() github.com/cockroachdb/cockroach/pkg/server/status.go:2871 +0x45 github.com/cockroachdb/cockroach/pkg/server.(*statusServer).CancelQueryByKey(0xc000a52c80, {0x63c85b8, 0xc0566148a0}, 0xc03f16a210) github.com/cockroachdb/cockroach/pkg/server/status.go:2882 +0x1aa github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_CancelQueryByKey_Handler.func1({0x63c85b8, 0xc0566148a0}, {0x4d07ba0, 0xc03f16a210}) github.com/cockroachdb/cockroach/pkg/server/serverpb/bazel-out/k8-opt/bin/pkg/server/serverpb/serverpb_go_proto_/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:8141 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}, 0xc0ccf7fc80, 0xc038161cf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:155 +0x643 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}, 0x3, 0xc071e54c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02790d980}}}, {0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}, 0xc0ccf7fc80, 0xc071e54c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056614810}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614810}, {0x40, 0xc02790db70}, 0xc02790dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056614810}, {0x4d07ba0, 0xc03f16a210}, 0xc0ccf7fc80, 0xc038161cf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/server/serverpb._Status_CancelQueryByKey_Handler({0x4eaf760, 0xc000a52c80}, {0x63c85b8, 0xc056614810}, 0xc071982ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/server/serverpb/bazel-out/k8-opt/bin/pkg/server/serverpb/serverpb_go_proto_/github.com/cockroachdb/cockroach/pkg/server/serverpb/status.pb.go:8143 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2b40, 0xc00cbb1560, 0x91daa68, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812775 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812589 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802768 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04ee0ad50}, 0xc049760528, 0x1, 0xc055c050a8, 0x519bbc, 0xc03fc42420, 0xc040912f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04ee0ad50}, 0x1e0, 0xc055c05230, 0x3fc423c0, 0x0, 0x660069, 0xc03fc42420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04ee0ad50}, 0xc03fc423c0, {{{{0x0, 0x0}, {0xc04f74ad80, 0xd3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04ee0ad50}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04ee0ad50}, {0xc044c5e120, {0x174301102754af9c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04ee0ad50}, 0xc0748e2500, 0xc055c059f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ee0ad50}, 0x660089, {0xc044c5e120, {0x174301102754af9c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04ee0ad50}, 0xc077381700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102754af9c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102754af9c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ee0ac90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ee0ac90}, {0xc055c073f8, 0x660089}, 0xc055c07400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ee0ac90}, {0x7fe3c2fa2cc0}, 0xc077381500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ee0ac00}, 0xc077381500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}, 0xc08f016480, 0xc028725170) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}, 0x3, 0xc04ccb1c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055c07980}}}, {0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}, 0xc08f016480, 0xc04ccb1c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ee0ac00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ee0ac00}, {0x40, 0xc055c07b70}, 0xc055c07ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ee0ac00}, {0x4e9d7a0, 0xc077381500}, 0xc08f016480, 0xc028725170) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ee0ac00}, 0xc01607e300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc044c5e000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc044c5e000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219783719 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc04ed25970, {0x63c85b8, 0xc035375920}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc084ec5900, {0x63c85b8, 0xc035375920}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc084ec5900, {0x63c85b8, 0xc0353758c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc084ec5900, {0x63c85b8, 0xc0353758c0}, 0xc0cdc056e0, {0x7fe42b790cf0, 0xc002a11bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc084ec5900, {0x63c85b8, 0xc043352330}, {{0x63f88f8, 0xc0cdc056e0}, {0xc07188bb9b, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc084ec5900, {0x63c85b8, 0xc043352330}, {{0x63f88f8, 0xc0cdc056e0}, {0xc07188bb9b, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0cdc056e0}, {0xc07188bb9b, 0x23}, 0x0, 0x0}, {0x25c60c0f, 0xedb7a8350, 0x0}, {0x25c61208, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc084ec5900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc084ec5900, {0x63c8510, 0xc06f04d500}, 0xc058f23dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0459e77f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06f04d500}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0459e77f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812605 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc04f63ad50}, 0xc072326fc0, 0x1, 0xc0408750a8, 0x519bbc, 0xc063465e00, 0xc08c1e5500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc067748000, {0x63c85b8, 0xc04f63ad50}, 0xc067748060, 0xc0408751a8, 0x41, 0x60, 0xffffffffffffffff, 0xc063465e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc04f63ad50}, 0xc063465d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc073a6a360, 0x1f}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f63ad50}, 0x6352260, 0x47c525c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc04f63ad50}, {0xc063465c20, {0x17430110576c2c1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc04f63ad50}, 0xc03ecb7400, 0xc0408759f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f63ad50}, 0x660089, {0xc063465c20, {0x17430110576c2c1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc04f63ad50}, 0xc0455a2400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110576c2c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110576c2c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63acc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63acc0}, {0xc0408773f8, 0x660089}, 0xc040877400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63acc0}, {0x1ffffffffe}, 0xc0455a2200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63ac30}, 0xc0455a2200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}, 0xc03ebdd360, 0xc031cfadf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}, 0x3, 0xc033d33400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc040877980}}}, {0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}, 0xc03ebdd360, 0xc033d33400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63ac30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63ac30}, {0x40, 0xc040877b70}, 0xc040877ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63ac30}, {0x4e9d7a0, 0xc0455a2200}, 0xc03ebdd360, 0xc031cfadf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63ac30}, 0xc02a97fc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e0657a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e0657a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 218904268 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1440, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0634b2980, 0xc004220000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0634b2980, {0xc004220000, 0xefbf, 0xefbf}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0634b2980, {0xc004220000, 0xefbf, 0xc04cdf9180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01b1239c0, {0xc004220000, 0x7eda79, 0xc004220005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc02cc6f920, {0xc004220000, 0x3, 0xefbf}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc075d0eac8, {0xc004220000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03d7e1078, {0x631b900, 0xc075d0eac8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03d7e0e00, {0x7fe42ab59310, 0xc02cc6f920}, 0xefbf) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03d7e0e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc03d7e0e00, {0xc0a9f10000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02eddfdb8, {0xc0a9f10000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04f8c4b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04f8c4b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04f8c50e0, {0x634aac0, 0xc04f8c4b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04f8c4a80, 0xc084984250, 0xc0663ef42b, 0xc0663ef42d, {0x635bec0, 0xc076601050}, 0xc000d6ae00, 0xc064744188, 0xc0663ef429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04f8c4a80, {0x63c85b8, 0xc017dc6210}, 0xc084984240, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0a9f0a037, 0x8}}, 0x0, 0xc017dc61b0, 0xc017dc61e0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc017dc6210}, {0x643d810, 0xc02cc6f920}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03b796e70}, {0x643d810, 0xc02cc6f920}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03b796e70}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807033 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc07b2009f0}, 0xc045b8db18, 0x1, 0xc0054750a8, 0x519bbc, 0xc06224d8c0, 0xc079ff0d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07b2009f0}, 0x1e0, 0xc005475230, 0x6224d860, 0x0, 0x660069, 0xc06224d8c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc07b2009f0}, 0xc06224d860, {{{{0x0, 0x0}, {0xc028ef4240, 0x116}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07b2009f0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc07b2009f0}, {0xc05208f8c0, {0x174301105504db5c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc07b2009f0}, 0xc06810f680, 0xc0054759f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07b2009f0}, 0x660089, {0xc05208f8c0, {0x174301105504db5c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc07b2009f0}, 0xc086429300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105504db5c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105504db5c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07b200960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b200960}, {0xc0054773f8, 0x660089}, 0xc005477400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07b200960}, {0x408c37eb1b156869}, 0xc086429100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07b2008d0}, 0xc086429100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}, 0xc0491011e0, 0xc048fe5320) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}, 0x3, 0xc05b422f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005477980}}}, {0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}, 0xc0491011e0, 0xc05b422f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07b2008d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b2008d0}, {0x40, 0xc005477b70}, 0xc005477ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07b2008d0}, {0x4e9d7a0, 0xc086429100}, 0xc0491011e0, 0xc048fe5320) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07b2008d0}, 0xc08efd0c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0592d0c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0592d0c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805043 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc0701d1620}, 0xc01fb0adf8, 0x1, 0x708017da5bc1fb, 0x37e11d600, 0xc070651b60, 0xc06b794960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0701d1620}, 0x120, 0xc078909230, 0x70651b00, 0xc03d0ba820, 0x18, 0xc070651b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc0701d1620}, 0xc070651b00, {{{{0x0, 0x0}, {0xc0557bc900, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0701d1620}, 0x6352410, 0x34331d70) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc0701d1620}, {0xc0706519e0, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc0701d1620}, 0xc03c95e0a0, 0xc0789099f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0701d1620}, 0x660089, {0xc0706519e0, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc0701d1620}, 0xc063fc1200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0701d1470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0701d1470}, {0xc07890b3f8, 0x660089}, 0xc07890b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0701d1470}, {0xc0015221e0}, 0xc063fc1000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0701d13b0}, 0xc063fc1000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}, 0xc065a4b1a0, 0xc01fb0adc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}, 0x3, 0xc00e1e8b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07890b980}}}, {0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}, 0xc065a4b1a0, 0xc00e1e8b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0701d13b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0701d13b0}, {0x40, 0xc07890bb70}, 0xc07890bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}, 0x203003, 0x203003) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0701d13b0}, {0x4e9d7a0, 0xc063fc1000}, 0xc065a4b1a0, 0xc01fb0adc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0701d13b0}, 0xc01838e5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806064 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc03c07c9c0}, 0xc08aad9008, 0x1, 0xc00768f0a8, 0x519bbc, 0xc038c6e060, 0xc0439e2ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03c07c9c0}, 0x120, 0xc00768f230, 0x38c6e000, 0x0, 0x660069, 0xc038c6e060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc03c07c9c0}, 0xc038c6e000, {{{{0x0, 0x0}, {0xc05adacfc0, 0x1f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03c07c9c0}, 0x6352410, 0x17e2edb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc03c07c9c0}, {0xc0784b7e60, {0x174301104ba6f35b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc03c07c9c0}, 0xc03289ab40, 0xc00768f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03c07c9c0}, 0x660089, {0xc0784b7e60, {0x174301104ba6f35b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc03c07c9c0}, 0xc06749f500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104ba6f35b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104ba6f35b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c07c900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c07c900}, {0xc0076913f8, 0x660089}, 0xc007691400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c07c900}, {0x0}, 0xc06749f200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c07c840}, 0xc06749f200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}, 0xc076b10220, 0xc08aad9248) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}, 0x3, 0xc027088e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007691980}}}, {0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}, 0xc076b10220, 0xc027088e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03c07c840}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c07c840}, {0x40, 0xc007691b70}, 0xc007691ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03c07c840}, {0x4e9d7a0, 0xc06749f200}, 0xc076b10220, 0xc08aad9248) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03c07c840}, 0xc01d942d20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0784b7d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0784b7d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809435 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02adf4b40, {0x63c85b8, 0xc070383a40}, 0xc06c8712a8, 0x1, 0xc026ddd0a8, 0x519bbc, 0xc08a001380, 0xc051eb5920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc070383a40}, 0x120, 0xc026ddd230, 0x8a001320, 0x0, 0x660069, 0xc08a001380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02adf4b40, {0x63c85b8, 0xc070383a40}, 0xc08a001320, {{{{0x0, 0x0}, {0xc036cb6ea0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc070383a40}, 0x6352410, 0x4b1bc938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ad47c70, {0x63c85b8, 0xc070383a40}, {0xc08a001200, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ad47c70, {0x63c85b8, 0xc070383a40}, 0xc0489f43c0, 0xc026ddd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc070383a40}, 0x660089, {0xc08a001200, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ae0b000, {0x63c85b8, 0xc070383a40}, 0xc0600a9300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070383980}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070383980}, {0xc026ddf3f8, 0x660089}, 0xc026ddf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070383980}, {0x40c4e4ffe07eb76c}, 0xc0600a9100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070383830}, 0xc0600a9100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}, 0xc0405a3c00, 0xc03f1805a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}, 0x3, 0xc0be9c21c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026ddf980}}}, {0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}, 0xc0405a3c00, 0xc0be9c21c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070383830}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070383830}, {0x40, 0xc026ddfb70}, 0xc026ddfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070383830}, {0x4e9d7a0, 0xc0600a9100}, 0xc0405a3c00, 0xc03f1805a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070383830}, 0xc03674bb60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08a0010e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08a0010e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801997 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc05387fcc0, {0x63c85b8, 0xc0bb638360}, 0xc028ace2e8, 0x1, 0xc01012b0a8, 0x519bbc, 0xc05ee14960, 0xc023e9c180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0bb638360}, 0x120, 0xc01012b230, 0x5ee14900, 0x0, 0x660069, 0xc05ee14960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc05387fcc0, {0x63c85b8, 0xc0bb638360}, 0xc05ee14900, {{{{0x0, 0x0}, {0xc03cfe9560, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0bb638360}, 0x6352410, 0x32339990) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc051ee6280, {0x63c85b8, 0xc0bb638360}, {0xc05ee147e0, {0x174301101f4b11dc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc051ee6280, {0x63c85b8, 0xc0bb638360}, 0xc02c9ff720, 0xc01012b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0bb638360}, 0x660089, {0xc05ee147e0, {0x174301101f4b11dc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05004cc00, {0x63c85b8, 0xc0bb638360}, 0xc027a74f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101f4b11dc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101f4b11dc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0bb6382d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0bb6382d0}, {0xc01012d3f8, 0x660089}, 0xc01012d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0bb6382d0}, {0x7fe3f72f0cb8}, 0xc027a74d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0bb638240}, 0xc027a74d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}, 0xc03c8c38e0, 0xc027eb5cf8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}, 0x3, 0xc01d24be00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01012d980}}}, {0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}, 0xc03c8c38e0, 0xc01d24be00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0bb638240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0bb638240}, {0x40, 0xc01012db70}, 0xc01012dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0bb638240}, {0x4e9d7a0, 0xc027a74d00}, 0xc03c8c38e0, 0xc027eb5cf8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0bb638240}, 0xc028743920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ee146c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ee146c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809230 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc01618c150}, 0xc0506a7c20, 0x1, 0xc0051cd0a8, 0x519bbc, 0xc05330ade0, 0xc07b689140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01618c150}, 0x120, 0xc0051cd230, 0x5330ad80, 0x0, 0x660069, 0xc05330ade0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc01618c150}, 0xc05330ad80, {{{{0x0, 0x0}, {0xc0564f5320, 0x1d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01618c150}, 0x6352410, 0x42a0f020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc01618c150}, {0xc05330ac60, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc01618c150}, 0xc073142dc0, 0xc0051cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01618c150}, 0x660089, {0xc05330ac60, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc01618c150}, 0xc0523d4700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01618c000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01618c000}, {0xc0051cf3f8, 0x660089}, 0xc0051cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01618c000}, {0x7fe3e0f68688}, 0xc0523d4500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03729def0}, 0xc0523d4500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}, 0xc02d9d6b40, 0xc0439450e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}, 0x3, 0xc03f78b580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0051cf980}}}, {0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}, 0xc02d9d6b40, 0xc03f78b580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03729def0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03729def0}, {0x40, 0xc0051cfb70}, 0xc0051cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03729def0}, {0x4e9d7a0, 0xc0523d4500}, 0xc02d9d6b40, 0xc0439450e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03729def0}, 0xc0242a32c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330ab40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330ab40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808080 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021e1cdc0, {0x63c85b8, 0xc08a603b30}, 0xc0cd864c00, 0x1, 0x0, 0x1, 0xc072f774a0, 0xc0733c7380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08a603b30}, 0x120, 0xc00bcc8728, 0x72f77440, 0x0, 0x660069, 0xc072f774a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021e1cdc0, {0x63c85b8, 0xc08a603b30}, 0xc072f77440, {{{{0x0, 0x0}, {0xc073120c60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08a603b30}, 0x6352410, 0x32431098) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021ef08c0, {0x63c85b8, 0xc08a603b30}, {0xc014f22ea0, {0x174301105a3a7b3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021ef08c0, {0x63c85b8, 0xc08a603b30}, 0xc06034b9a0, 0xc078182ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08a603b30}, 0x660089, {0xc014f22ea0, {0x174301105a3a7b3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021f8c800, {0x63c85b8, 0xc08a603b30}, 0xc07babdb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a3a7b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a3a7b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08a603aa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a603aa0}, {0xc09365a8f0, 0x660089}, 0xc09365a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08a603aa0}, {0xc00102e4e0}, 0xc07babda00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08a603a40}, 0xc07babda00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0478963c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806424 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc05ec99c00, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc075769138, 0xc09296ee38, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc09296ef50, 0x0, 0xc09296ef50) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc09296ede8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09296ede8, {0xc00b82a870, 0x19, 0x4a}, {0xc00213dd50, 0x19, 0x6f}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09296dbc0, {0xc00213dd50, 0x0, 0xc075769480}, 0x121ea30, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09296dbc0, {0xc00b82a870, 0x19, 0x6f}, {0xc00213dd50, 0x18, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09296d900, {0xc00213dd50, 0x19, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc09cd50c08, {{0xc041a01e60, 0x18, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc08b702600, {{0xc041a01e60, 0x18, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04cefc400, {0x63c85b8, 0xc048370300}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc048370300}, {0x64a6560, 0xc08b702600}, {0xc041a01e60, 0x0, 0x30}, {0x174301109ecc4f2d, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc048370300}, {0x6466ed8, 0xc09cd50c00}, {0xc041a01e60, 0x18, 0x30}, {0x174301109ecc4f2d, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc065d80348}, {{0x174301109ecc4f2d, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109ecc4f2d, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0a085ef00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bc99b42d, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d6400, {0x63c85b8, 0xc048370300}, 0xc0a085ef00, 0xc02517d900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d6400, {0x63c85b8, 0xc048370300}, 0xc0a085ef00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109ecc4f2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109ecc4f2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048370240}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048370240}, {0xc07576f398, 0x660089}, 0xc07576f3a0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048370240}, {0xc00102e4e0}, 0xc0a085ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048370150}, 0xc0a085ee00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc00e1be1e0, {0x63c85b8, 0xc048370060}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc00e1be1e0, {0x63c85b8, 0xc048370060}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc058299310, {0x63c85b8, 0xc048370060}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0520d0a80, {0x63c85b8, 0xc048370060}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc02b36ee60) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0650cd380) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc0288fdec0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/is_null_ops.eg.go:226 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc038d62ea0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0bc28e300) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc06deacb00, {0x63c8510, 0xc0650cd340}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc08d374460, {0x63c8510, 0xc0650cd340}, 0xc073b573f0) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc0650cd040}, 0xc02195cb60, 0xc073b573f0, 0xc0645b8e00, 0xc0386f4000, 0xc054d275e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc0650cd040}, 0xc054d272f0, 0xc02195cb60, 0xc01a6dd140, {{0x63ca960, 0xc02593e140}, 0x0}, 0xc0386f4000) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc054d26c00, {0x63c8510, 0xc0650cd040}, 0xc054d272f0, 0xc0650cd040, {0x7fe42b790cf0, 0xc01a6dd140}, 0x0, 0xc038d62cb8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc054d26c00, {0x63c8510, 0xc0650cd040}, 0xc054d272f0, {0x7fe42b790cf0, 0xc01a6dd140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc054d26c00, {0x63c85b8, 0xc0288fd350}, {{0x63f8d78, 0xc08dda5400}, {0xc086c345dd, 0xd7}, 0x1, 0x1}, 0xc013054fc0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc054d26c00, {0x63c85b8, 0xc0288fd350}, {{0x63f8d78, 0xc08dda5400}, {0xc086c345dd, 0xd7}, 0x1, 0x1}, 0xc013054fc0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc054d26c00, {0x63c85b8, 0xc0288fd350}, {0xc013054fc0, {0xc04b28da40, 0x1, 0x1}, {0xc03edbaaa0, 0x6, 0x6}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc086c34719, 0x0}, 0x0, {0xc5a2129, 0xedb7a8351, 0x0}, 0x1}, 0xc054d26c00, 0xc075775980, 0xc075775970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc054d26c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc054d26c00, {0x63c8510, 0xc079baaf80}, 0xc076d8fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc051ffdad0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc079baaf80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc051ffdad0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804589 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc042935020}, 0xc0996ff008, 0x1, 0xc076ef50a8, 0x519bbc, 0xc051697320, 0xc0473596e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc042935020}, 0x1e0, 0xc076ef5230, 0x516972c0, 0x0, 0x660069, 0xc051697320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc042935020}, 0xc0516972c0, {{{{0x0, 0x0}, {0xc07c2d3b00, 0x54}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc042935020}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc042935020}, {0xc060279320, {0x1743011031e4a909, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc042935020}, 0xc0667474a0, 0xc076ef59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc042935020}, 0x660089, {0xc060279320, {0x1743011031e4a909, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc042935020}, 0xc076d2d800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042934f90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042934f90}, {0xc076ef73f8, 0x660089}, 0xc076ef7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042934f90}, {0x0}, 0xc076d2d500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042934f00}, 0xc076d2d500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}, 0xc0994b55c0, 0xc02e7132d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}, 0x3, 0xc050a939c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076ef7980}}}, {0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}, 0xc0994b55c0, 0xc050a939c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc042934f00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042934f00}, {0x40, 0xc076ef7b70}, 0xc076ef7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc042934f00}, {0x4e9d7a0, 0xc076d2d500}, 0xc0994b55c0, 0xc02e7132d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc042934f00}, 0xc028849800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060279200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060279200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804675 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc017b16c80, {0x63c85b8, 0xc079174870}, 0xc026b95d58, 0x1, 0xc0624430a8, 0x519bbc, 0xc06475aa80, 0xc05185d6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc079174870}, 0x120, 0xc062443230, 0x6475aa20, 0x0, 0x660069, 0xc06475aa80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc017b16c80, {0x63c85b8, 0xc079174870}, 0xc06475aa20, {{{{0x0, 0x0}, {0xc0baba4900, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc079174870}, 0x6352410, 0x3d0ac500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc008447bd0, {0x63c85b8, 0xc079174870}, {0xc06475a900, {0x1743011031e4a909, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc008447bd0, {0x63c85b8, 0xc079174870}, 0xc075406140, 0xc0624439f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc079174870}, 0x660089, {0xc06475a900, {0x1743011031e4a909, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000e39800, {0x63c85b8, 0xc079174870}, 0xc02a22bd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0791747e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0791747e0}, {0xc0624453f8, 0x660089}, 0xc062445400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0791747e0}, {0xc0015221e0}, 0xc02a22bb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079174750}, 0xc02a22bb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}, 0xc054287f20, 0xc048fc7cc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}, 0x3, 0xc06d761680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062445980}}}, {0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}, 0xc054287f20, 0xc06d761680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc079174750}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079174750}, {0x40, 0xc062445b70}, 0xc062445ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc079174750}, {0x4e9d7a0, 0xc02a22bb00}, 0xc054287f20, 0xc048fc7cc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc079174750}, 0xc0abc70b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06475a7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06475a7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809588 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc030af5900, {0x63c85b8, 0xc070383410}, 0xc052163920, 0x1, 0xc026a070a8, 0x519bbc, 0xc08a001020, 0xc084b32060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc070383410}, 0x120, 0xc026a07230, 0x8a000fc0, 0x0, 0x660069, 0xc08a001020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc030af5900, {0x63c85b8, 0xc070383410}, 0xc08a000fc0, {{{{0x0, 0x0}, {0xc05abc0000, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc070383410}, 0x6352410, 0x42a0f470) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0308a0870, {0x63c85b8, 0xc070383410}, {0xc088160000, {0x1743011064ff58cd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0308a0870, {0x63c85b8, 0xc070383410}, 0xc09854ce60, 0xc026a079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc070383410}, 0x660089, {0xc088160000, {0x1743011064ff58cd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030b32c00, {0x63c85b8, 0xc070383410}, 0xc0600a9000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011064ff58cd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011064ff58cd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0682a96b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0682a96b0}, {0xc026a093f8, 0x660089}, 0xc026a09400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0682a96b0}, {0xc0015221e0}, 0xc058b31500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0682a9620}, 0xc058b31500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}, 0xc0472a2ec0, 0xc0337f8d08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}, 0x3, 0xc01b65d400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026a09980}}}, {0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}, 0xc0472a2ec0, 0xc01b65d400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0682a9620}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0682a9620}, {0x40, 0xc026a09b70}, 0xc026a09ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0682a9620}, {0x4e9d7a0, 0xc058b31500}, 0xc0472a2ec0, 0xc0337f8d08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0682a9620}, 0xc043ace180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07c3be6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07c3be6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219282980 [IO wait]: internal/poll.runtime_pollWait(0x7fdb124bc6d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc02ebb4c00, 0xc042456000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02ebb4c00, {0xc042456000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc02ebb4c00, {0xc042456000, 0x735, 0xc038cdbcc0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03080f200, {0xc042456000, 0x7eda79, 0xc042456005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc048d0d800, {0xc042456000, 0x4bf403, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc047918768, {0xc042456000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06f1b9af8, {0x631b900, 0xc047918768}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06f1b9880, {0x7fe42ab59310, 0xc048d0d800}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06f1b9880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06f1b9880, {0xc08ac48000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02605e828, {0xc08ac48000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc073314088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc073314088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc073314660, {0x634aac0, 0xc073314088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc073314000, 0xc07a89fae0, 0xc057c7b42b, 0xc057c7b42d, {0x635bec0, 0xc06485d170}, 0xc000d6ae00, 0xc03ae85908, 0xc057c7b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc073314000, {0x63c85b8, 0xc066fbe780}, 0xc07a89fad0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc078589021, 0x8}}, 0x0, 0xc066fbe720, 0xc066fbe750, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc066fbe780}, {0x643d810, 0xc048d0d800}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0731c6600}, {0x643d810, 0xc048d0d800}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0731c6600}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800805 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0b2c00, {0x63c85b8, 0xc0374be7e0}, 0xc055603600, 0xc066747b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc0374be7e0}, 0xc055603600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff25f28bf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff25f28bf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0374be720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0374be720}, {0xc047bfd3f8, 0x660089}, 0xc047bfd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0374be720}, {0xc0015221e0}, 0xc055603400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0374be600}, 0xc055603400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}, 0xc033d65080, 0xc067a60660) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}, 0x3, 0xc0038d8bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc047bfd980}}}, {0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}, 0xc033d65080, 0xc0038d8bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0374be600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0374be600}, {0x40, 0xc047bfdb70}, 0xc047bfdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0374be600}, {0x4e9d7a0, 0xc055603400}, 0xc033d65080, 0xc067a60660) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0374be600}, 0xc01f546000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc068b39320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc068b39320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804489 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc046bee300}, 0xc02339ae10, 0x1, 0xc0790710a8, 0x519bbc, 0xc04eda7320, 0xc0403416e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc046bee300}, 0x1e0, 0xc079071230, 0x4eda72c0, 0x0, 0x660069, 0xc04eda7320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc046bee300}, 0xc04eda72c0, {{{{0x0, 0x0}, {0xc01fe83440, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc046bee300}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc046bee300}, {0xc05f3b4000, {0x17430110311253a3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc046bee300}, 0xc0195d3e00, 0xc0790719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046bee300}, 0x660089, {0xc05f3b4000, {0x17430110311253a3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc046bee300}, 0xc0626e9c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110311253a3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110311253a3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046bee270}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046bee270}, {0xc0790733f8, 0x660089}, 0xc079073400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046bee270}, {0x0}, 0xc0626e9a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046bee1e0}, 0xc0626e9a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}, 0xc08cfa6ec0, 0xc02d328918) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}, 0x3, 0xc0585874c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079073980}}}, {0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}, 0xc08cfa6ec0, 0xc0585874c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046bee1e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046bee1e0}, {0x40, 0xc079073b70}, 0xc079073ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046bee1e0}, {0x4e9d7a0, 0xc0626e9a00}, 0xc08cfa6ec0, 0xc02d328918) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046bee1e0}, 0xc01aa3d740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05eaf9e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05eaf9e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811349 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ffb7c00, {0x63c85b8, 0xc083dbfc20}, 0xc055383100, 0xc088618be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ffb7c00, {0x63c85b8, 0xc083dbfc20}, 0xc055383100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f0dab1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f0dab1c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc083dbfb90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc083dbfb90}, {0xc0199e73f8, 0x660089}, 0xc0199e7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc083dbfb90}, {0x0}, 0xc055382e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc083dbfb00}, 0xc055382e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}, 0xc037ae3a80, 0xc02dde19f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}, 0x3, 0xc06f257200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0199e7980}}}, {0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}, 0xc037ae3a80, 0xc06f257200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc083dbfb00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc083dbfb00}, {0x40, 0xc0199e7b70}, 0xc0199e7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc083dbfb00}, {0x4e9d7a0, 0xc055382e00}, 0xc037ae3a80, 0xc02dde19f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc083dbfb00}, 0xc04b461b60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b4dcd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b4dcd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812524 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806884 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021a5f540, {0x63c85b8, 0xc0919c6db0}, 0xc03c2e9908, 0x1, 0xc003ebf0a8, 0x519bbc, 0xc04cb6a180, 0xc056f28ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0919c6db0}, 0x120, 0xc003ebf230, 0x4cb6a120, 0x0, 0x660069, 0xc04cb6a180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021a5f540, {0x63c85b8, 0xc0919c6db0}, 0xc04cb6a120, {{{{0x0, 0x0}, {0xc07398dd40, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0919c6db0}, 0x6352410, 0x25ab6330) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a30e60, {0x63c85b8, 0xc0919c6db0}, {0xc04cb6a000, {0x174301104efe0834, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a30e60, {0x63c85b8, 0xc0919c6db0}, 0xc0b7068960, 0xc003ebf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0919c6db0}, 0x660089, {0xc04cb6a000, {0x174301104efe0834, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b2800, {0x63c85b8, 0xc0919c6db0}, 0xc097d62100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104efe0834, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104efe0834, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0919c6d20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0919c6d20}, {0xc003ec13f8, 0x660089}, 0xc003ec1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0919c6d20}, {0x7fde18100dc8}, 0xc0981b5f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0919c6c90}, 0xc0981b5f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}, 0xc08e4eac40, 0xc02d6d0ea0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}, 0x3, 0xc08e2dd4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003ec1980}}}, {0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}, 0xc08e4eac40, 0xc08e2dd4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0919c6c90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0919c6c90}, {0x40, 0xc003ec1b70}, 0xc003ec1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0919c6c90}, {0x4e9d7a0, 0xc0981b5f00}, 0xc08e4eac40, 0xc02d6d0ea0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0919c6c90}, 0xc036d36540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0689c7e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0689c7e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811899 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035900, {0x63c85b8, 0xc01acfec00}, 0xc063fafb00, 0x1, 0x7080181a245dcf, 0x37e11d600, 0xc04cabbda0, 0xc03f1d1ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01acfec00}, 0x120, 0xc015e7f230, 0x4cabbd40, 0x0, 0x660069, 0xc04cabbda0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035900, {0x63c85b8, 0xc01acfec00}, 0xc04cabbd40, {{{{0x0, 0x0}, {0xc00da77320, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01acfec00}, 0x6352410, 0x6b0668a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd810, {0x63c85b8, 0xc01acfec00}, {0xc04cabbc20, {0x174301107b1cf270, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd810, {0x63c85b8, 0xc01acfec00}, 0xc0761ed0e0, 0xc015e7f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01acfec00}, 0x660089, {0xc04cabbc20, {0x174301107b1cf270, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06d400, {0x63c85b8, 0xc01acfec00}, 0xc0d29ed700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b1cf270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b1cf270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01acfeb70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01acfeb70}, {0xc015e813f8, 0x660089}, 0xc015e81400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01acfeb70}, {0x7fe3e21dbb88}, 0xc0d29ed500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01acfea80}, 0xc0d29ed500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}, 0xc04c9861a0, 0xc063fafad0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}, 0x3, 0xc05426d940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc015e81980}}}, {0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}, 0xc04c9861a0, 0xc05426d940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01acfea80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01acfea80}, {0x40, 0xc015e81b70}, 0xc015e81ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01acfea80}, {0x4e9d7a0, 0xc0d29ed500}, 0xc04c9861a0, 0xc063fafad0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01acfea80}, 0xc02f147560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04cabbb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc04cabbb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804876 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0310a0c00, {0x63c85b8, 0xc041dc1380}, 0xc0878b1100, 0xc044f78140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310a0c00, {0x63c85b8, 0xc041dc1380}, 0xc0878b1100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b9aa249, 0x0, 0x0}, 0xc09f307f90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b9aa249, 0x0, 0x0}, 0xc09f307f90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041dc12f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041dc12f0}, {0xc057c75918, 0x660089}, 0xc057c75920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041dc12f0}, {0xc00102e4e0}, 0xc0878b1000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041dc1290}, 0xc0878b1000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc02e6747e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc041dc1200}, {0xc00dfd36b0, 0x24}, 0x46c77bee6, 0xc028b32d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x10, 0xc0199d0478}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc041dc1200}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809919 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02efd4140, {0x63c85b8, 0xc024dc8930}, 0xc02765e120, 0x1, 0xc00eecb0a8, 0x519bbc, 0xc00f5314a0, 0xc07b854cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc024dc8930}, 0x120, 0xc00eecb230, 0xf531440, 0x0, 0x660069, 0xc00f5314a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02efd4140, {0x63c85b8, 0xc024dc8930}, 0xc00f531440, {{{{0x0, 0x0}, {0xc05aa5e480, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc024dc8930}, 0x6352410, 0x7818e0c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ef76e60, {0x63c85b8, 0xc024dc8930}, {0xc00f531320, {0x1743011069d0f3b9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ef76e60, {0x63c85b8, 0xc024dc8930}, 0xc024e00c80, 0xc00eecb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024dc8930}, 0x660089, {0xc00f531320, {0x1743011069d0f3b9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02efcb000, {0x63c85b8, 0xc024dc8930}, 0xc045d54500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011069d0f3b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011069d0f3b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024dc88a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024dc88a0}, {0xc00eecd3f8, 0x660089}, 0xc00eecd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024dc88a0}, {0x7fe38e2e2b98}, 0xc08205bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024dc8810}, 0xc08205bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}, 0xc0254237c0, 0xc02765f308) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}, 0x3, 0xc02941b200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00eecd980}}}, {0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}, 0xc0254237c0, 0xc02941b200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc024dc8810}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024dc8810}, {0x40, 0xc00eecdb70}, 0xc00eecdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc024dc8810}, {0x4e9d7a0, 0xc08205bf00}, 0xc0254237c0, 0xc02765f308) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc024dc8810}, 0xc035f09c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00f531200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc00f531200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812762 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810875 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021d552c0, {0x63c85b8, 0xc07527ac90}, 0xc01f4473e0, 0x1, 0x0, 0x1, 0xc04a44a960, 0xc06cd88de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07527ac90}, 0x120, 0xc010bb4728, 0x4a44a900, 0x0, 0x660069, 0xc04a44a960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021d552c0, {0x63c85b8, 0xc07527ac90}, 0xc04a44a900, {{{{0x0, 0x0}, {0xc08511ab40, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07527ac90}, 0x6352410, 0x51b89438) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3cbe0, {0x63c85b8, 0xc07527ac90}, {0xc04a44a480, {0x174301106bc25ef1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3cbe0, {0x63c85b8, 0xc07527ac90}, 0xc0cbeb48c0, 0xc081e5cee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07527ac90}, 0x660089, {0xc04a44a480, {0x174301106bc25ef1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021daa000, {0x63c85b8, 0xc07527ac90}, 0xc08881d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bc25ef1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bc25ef1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07527a900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07527a900}, {0xc0076ae8f0, 0x660089}, 0xc0076ae8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07527a900}, {0xc00102e4e0}, 0xc08881d300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07527a840}, 0xc08881d300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03c175c80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802504 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc058a20720}, 0xc0b87c8870, 0x1, 0xc055cc30a8, 0x519bbc, 0xc03c440f00, 0xc020a9cba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc058a20720}, 0x120, 0xc055cc3230, 0x3c440ea0, 0x0, 0x660069, 0xc03c440f00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc058a20720}, 0xc03c440ea0, {{{{0x0, 0x0}, {0xc048e04000, 0x4c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc058a20720}, 0x6352410, 0x1a06f840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc058a20720}, {0xc03c440d80, {0x174301102226f774, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc058a20720}, 0xc071524d20, 0xc055cc39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc058a20720}, 0x660089, {0xc03c440d80, {0x174301102226f774, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc058a20720}, 0xc03d8acb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102226f774, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102226f774, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc058a20690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058a20690}, {0xc055cc53f8, 0x660089}, 0xc055cc5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc058a20690}, {0x4ed525c}, 0xc03d8ac100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc058a20600}, 0xc03d8ac100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}, 0xc058ed13e0, 0xc02b55e750) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}, 0x3, 0xc0a32e1c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055cc5980}}}, {0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}, 0xc058ed13e0, 0xc0a32e1c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc058a20600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058a20600}, {0x40, 0xc055cc5b70}, 0xc055cc5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc058a20600}, {0x4e9d7a0, 0xc03d8ac100}, 0xc058ed13e0, 0xc02b55e750) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc058a20600}, 0xc0516792c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03c440c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03c440c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812730 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f7a4dc0, {0x63c85b8, 0xc062cdf080}, 0xc05940e138, 0x1, 0xc03a1c50a8, 0x519bbc, 0xc0588e8f60, 0xc06ca80420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc073c8e7e0, {0x63c85b8, 0xc062cdf080}, 0xc0588e8f60, 0xc03a1c51a8, 0x31, 0x40, 0x1, 0xc0588e8f60, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f7a4dc0, {0x63c85b8, 0xc062cdf080}, 0xc0588e8ea0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc03ee457a0, 0x6}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc062cdf080}, 0x6352260, 0x3abf6b20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f663040, {0x63c85b8, 0xc062cdf080}, {0xc0588e8c60, {0x174301105af607c5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f663040, {0x63c85b8, 0xc062cdf080}, 0xc05eff8f00, 0xc03a1c59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc062cdf080}, 0x660089, {0xc0588e8c60, {0x174301105af607c5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f7c3000, {0x63c85b8, 0xc062cdf080}, 0xc050ef3800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105af607c5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105af607c5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cdefc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdefc0}, {0xc03a1c73f8, 0x660089}, 0xc03a1c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cdefc0}, {0x7fe429f399d8}, 0xc050ef3600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cdef00}, 0xc050ef3600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}, 0xc06ac12ca0, 0xc0490962e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}, 0x3, 0xc0b08ec100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03a1c7980}}}, {0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}, 0xc06ac12ca0, 0xc0b08ec100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cdef00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdef00}, {0x40, 0xc03a1c7b70}, 0xc03a1c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cdef00}, {0x4e9d7a0, 0xc050ef3600}, 0xc06ac12ca0, 0xc0490962e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cdef00}, 0xc049257d40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f8900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f8900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811905 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc016c74030}, 0xc063fafe48, 0x1, 0xc028f0f0a8, 0x519bbc, 0xc019e57da0, 0xc040c2b5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc016c74030}, 0x120, 0xc028f0f230, 0x19e57d40, 0x0, 0x660069, 0xc019e57da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc016c74030}, 0xc019e57d40, {{{{0x0, 0x0}, {0xc04d1746c0, 0x69}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc016c74030}, 0x6352410, 0x6b066978) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc016c74030}, {0xc019e57c20, {0x174301107d502aaa, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc016c74030}, 0xc057f20a00, 0xc028f0f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc016c74030}, 0x660089, {0xc019e57c20, {0x174301107d502aaa, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc016c74030}, 0xc081bf7800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107d502aaa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107d502aaa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc015dabf50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015dabf50}, {0xc028f113f8, 0x660089}, 0xc028f11400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc015dabf50}, {0x7fe350be7780}, 0xc081bf7600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc015dabce0}, 0xc081bf7600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}, 0xc04c986f00, 0xc053344978) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}, 0x3, 0xc06b2037c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028f11980}}}, {0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}, 0xc04c986f00, 0xc06b2037c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc015dabce0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015dabce0}, {0x40, 0xc028f11b70}, 0xc028f11ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc015dabce0}, {0x4e9d7a0, 0xc081bf7600}, 0xc04c986f00, 0xc053344978) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc015dabce0}, 0xc0d223e360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc019e57b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc019e57b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806670 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a65d040, {0x63c85b8, 0xc09133e1b0}, 0xc049e1f350, 0x1, 0xc0035230a8, 0x519bbc, 0xc0a2e364a0, 0xc0352e09c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc073d850e0, {0x63c85b8, 0xc09133e1b0}, 0xc073d85130, 0xc0035231a8, 0x56, 0x60, 0xffffffffffffffff, 0xc0a2e364a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a65d040, {0x63c85b8, 0xc09133e1b0}, 0xc0a2e36380, {{{{0x0, 0x0}, {0xc044e307e0, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x60, {0x63c85b8, 0xc09133e1b0}, 0x0, 0xf) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a5d7950, {0x63c85b8, 0xc09133e1b0}, {0xc0633d50e0, {0x17430110514af0b8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a5d7950, {0x63c85b8, 0xc09133e1b0}, 0xc02506d5e0, 0xc0035239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc09133e1b0}, 0x660089, {0xc0633d50e0, {0x17430110514af0b8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a26d000, {0x63c85b8, 0xc09133e1b0}, 0xc0975ead00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110514af0b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110514af0b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09133e120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09133e120}, {0xc0035253f8, 0x660089}, 0xc003525400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09133e120}, {0x0}, 0xc0975eab00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09133e090}, 0xc0975eab00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}, 0xc08eb79dc0, 0xc049e1fc68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}, 0x3, 0xc08f0c8cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003525980}}}, {0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}, 0xc08eb79dc0, 0xc08f0c8cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09133e090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09133e090}, {0x40, 0xc003525b70}, 0xc003525ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09133e090}, {0x4e9d7a0, 0xc0975eab00}, 0xc08eb79dc0, 0xc049e1fc68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09133e090}, 0xc038ce7f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0633d4ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0633d4ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799943 [runnable]: sync.runtime_SemacquireMutex(0xc055260690, 0x8f, 0x7fe31ebc3e40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc055260538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc055260560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0c321d200, {0xffffffffffffff15, 0xc0552606c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0c321d200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0c321d200, {0xc005e41ab0, 0x32, 0x6d}, {0xc018956a00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0bf3e5298, {0xc005e41ab0, 0x32, 0x6d}, {0xc018956a00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0bf3e3bc0, {0xc018956a00, 0x74, 0x6d}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0bf3e3bc0, {0xc005e41ab0, 0x74, 0xc018956a00}, {0xc018956a00, 0xc06537b5f0, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0bf3e3900, {0xc018956a00, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc03fdcf8a8, {{0xc06537b5f0, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07a325b00, {{0xc0bab98ed0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc087fcc400, {0x63c85b8, 0xc0799d8a20}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0799d8a20}, {0x64a6560, 0xc07a325b00}, {0xc0bab98ed0, 0x76, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0799d8a20}, {0x6466ed8, 0xc03fdcf600}, {0xc0bab98ed0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9a5a0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc048bef000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209bf400, {0x63c85b8, 0xc0799d8a20}, 0xc048bef000, 0xc066747680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209bf400, {0x63c85b8, 0xc0799d8a20}, 0xc048bef000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0799d8900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0799d8900}, {0xc0076c88f0, 0x660089}, 0xc0076c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0799d8900}, {0xc00102e4e0}, 0xc048beed00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0799d88a0}, 0xc048beed00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01174b980}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219230325 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0537ed810, {0x63c85b8, 0xc0871a2fc0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0708a8c00, {0x63c85b8, 0xc0871a2fc0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0708a8c00, {0x63c8510, 0xc08532a300}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc0708a8c00, {0x63c8510, 0xc08532a300}, {0x63f7db8, 0x9bf6f48}, 0xc081ce9088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0708a8c00, {0x63c85b8, 0xc081c10ba0}, {{0x63f7db8, 0x9bf6f48}, {0xc084da4820, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0708a8c00, {0x63c85b8, 0xc081c10ba0}, {{0x63f7db8, 0x9bf6f48}, {0xc084da4820, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc084da4820, 0x6}, 0x0, 0x0}, {0x33685906, 0xedb7a834f, 0x0}, {0x33685ecf, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0708a8c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0708a8c00, {0x63c8510, 0xc036dcc4c0}, 0xc0b1cbbdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02cceb630) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc036dcc4c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02cceb630) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808865 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0580318d0, {0x63c85b8, 0xc0449b3dd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc0449b3dd0}, {0x4f67ad2, 0x6446ff8}, 0xc024601fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219803382 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc075bd8090}, 0xc0afa51c00, 0xc0627cc000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc075bd8090}, 0xc0afa51c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d1fa0b8, 0x0, 0x0}, 0xc029e8a6c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d1fa0b8, 0x0, 0x0}, 0xc029e8a6c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075bd8000}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075bd8000}, {0xc06e5c73f8, 0x660089}, 0xc06e5c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075bd8000}, {0xc01c2df7e8}, 0xc0afa51a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01d4d9f50}, 0xc0afa51a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}, 0xc035050200, 0xc0379d0810) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}, 0x3, 0xc020de60c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06e5c7980}}}, {0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}, 0xc035050200, 0xc020de60c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01d4d9f50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d4d9f50}, {0x40, 0xc06e5c7b70}, 0xc06e5c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01d4d9f50}, {0x4e9d7a0, 0xc0afa51a00}, 0xc035050200, 0xc0379d0810) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01d4d9f50}, 0xc02210ce40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03aa8aea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03aa8aea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811287 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc005c3fb00}, 0xc08d1b4e00, 0xc0652bad20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc005c3fb00}, 0xc08d1b4e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107478249c, 0x0, 0x0}, 0xc03cd41770, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107478249c, 0x0, 0x0}, 0xc03cd41770, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc005c3fa70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc005c3fa70}, {0xc0179673f8, 0x660089}, 0xc017967400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc005c3fa70}, {0x7fe392121498}, 0xc08d1b4c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc005c3f9e0}, 0xc08d1b4c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}, 0xc081b09f60, 0xc072280138) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}, 0x3, 0xc09ac19b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017967980}}}, {0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}, 0xc081b09f60, 0xc09ac19b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc005c3f9e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc005c3f9e0}, {0x40, 0xc017967b70}, 0xc017967ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc005c3f9e0}, {0x4e9d7a0, 0xc08d1b4c00}, 0xc081b09f60, 0xc072280138) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc005c3f9e0}, 0xc050204fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06e314900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06e314900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801549 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021ae4c00, {0x63c85b8, 0xc04173b8c0}, 0xc050943a00, 0xc07401f5e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021ae4c00, {0x63c85b8, 0xc04173b8c0}, 0xc050943a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101cf9ef38, 0x0, 0x0}, 0xc00d975ed0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101cf9ef38, 0x0, 0x0}, 0xc00d975ed0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04173b800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04173b800}, {0xc0766cf918, 0x660089}, 0xc0766cf920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04173b800}, {0xc00102e4e0}, 0xc050943900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04173b770}, 0xc050943900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc046cf6fc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04173b170}, {0xc00dfd36b0, 0x24}, 0x46c6f1c46, 0xc017292d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc07301e2d0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04173b170}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808696 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0185643c0, {0x63c85b8, 0xc0777bd080}, 0xc02ed27548, 0x1, 0xc0111a50a8, 0x519bbc, 0xc081e70960, 0xc0c7dc1c00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0564f5e60, {0x63c85b8, 0xc0777bd080}, 0xc0564f5ea0, 0xc0111a51a8, 0x16, 0x18, 0xffffffffffffffff, 0xc081e70960, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0185643c0, {0x63c85b8, 0xc0777bd080}, 0xc081e70900, {{{{0xc05a4837a0, 0x2}, {0xc0873f87e0, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0777bd080}, 0x6352230, 0x57084310) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0185322d0, {0x63c85b8, 0xc0777bd080}, {0xc081e707e0, {0x174301105eed94f1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0185322d0, {0x63c85b8, 0xc0777bd080}, 0xc0611343c0, 0xc0111a59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0777bd080}, 0x660089, {0xc081e707e0, {0x174301105eed94f1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018570c00, {0x63c85b8, 0xc0777bd080}, 0xc05ad1dc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105eed94f1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105eed94f1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0777bcff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0777bcff0}, {0xc0111a73f8, 0x660089}, 0xc0111a7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0777bcff0}, {0x7fe3262c3cc8}, 0xc05ad1d900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0777bcf30}, 0xc05ad1d900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}, 0xc0327ae680, 0xc021ff7458) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}, 0x3, 0xc0152b6a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0111a7980}}}, {0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}, 0xc0327ae680, 0xc0152b6a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0777bcf30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0777bcf30}, {0x40, 0xc0111a7b70}, 0xc0111a7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0777bcf30}, {0x4e9d7a0, 0xc05ad1d900}, 0xc0327ae680, 0xc021ff7458) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0777bcf30}, 0xc03abafc80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081e705a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081e705a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219149963 [select, 1 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 219800204 [semacquire]: sync.runtime_SemacquireMutex(0xc041e489c0, 0x3, 0xc041e489c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc078d9c338) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fe398188dc0, 0x7f0e, 0x7fd7688d7240) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc02551f080, 0x7fe398188d01, 0x7fe398188de0, 0x4149, 0x414e) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc025554580, {0x2572d33, 0x4149}, 0x0, 0xc06ac038f8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc06ac03680, {0xffffffffffffff15, 0xc078d9c5c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc06ac03680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc06ac03680, {0xc0602ed800, 0x2b, 0x59}, 0x13, 0x81) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc06ac03680, {0xc041dcaaf0, 0x2b, 0x59}, {0xc0602ed800, 0x51c9860, 0xc0602ed800}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc06ac03680, {0xc041dcaaf0, 0x2b, 0x63}, {0xc0602ed800, 0x2b, 0x59}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092e85298, {0xc041dcaaf0, 0x2b, 0x63}, {0xc0602ed800, 0x2b, 0x59}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092e83bc0, {0xc0602ed800, 0x59, 0x63}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092e83bc0, {0xc041dcaaf0, 0x2b, 0xc0602ed800}, {0xc0602ed800, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092e83900, {0xc0602ed800, 0x2b, 0x59}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc079a1d088, {{0xc04e27d440, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0262b3800, {{0xc04e27d440, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04e669c00, {0x63c85b8, 0xc022ab35f0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc022ab35f0}, {0x64a6560, 0xc0262b3800}, {0xc04e27d440, 0xb9e7b0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc022ab35f0}, {0x6466ed8, 0xc079a1d080}, {0xc04e27d440, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04d876e28}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04ad0e900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a34800, {0x63c85b8, 0xc022ab35f0}, 0xc04ad0e900, 0xc07442f220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a34800, {0x63c85b8, 0xc022ab35f0}, 0xc04ad0e900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022ab33e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022ab33e0}, {0xc0076ca8f0, 0x660089}, 0xc0076ca8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022ab33e0}, {0xc00102e4e0}, 0xc04ad0e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022ab3200}, 0xc04ad0e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc043b87800}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802709 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d22e3c0, {0x63c85b8, 0xc02c599ad0}, 0xc05f433008, 0x1, 0xc054fd50a8, 0x519bbc, 0xc04c615020, 0xc04dcf8e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02c599ad0}, 0x120, 0xc054fd5230, 0x4c614fc0, 0x0, 0x660069, 0xc04c615020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d22e3c0, {0x63c85b8, 0xc02c599ad0}, 0xc04c614fc0, {{{{0x0, 0x0}, {0xc052552fc0, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02c599ad0}, 0x6352230, 0x205886e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d195040, {0x63c85b8, 0xc02c599ad0}, {0xc04c614ea0, {0x1743011024887ab4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d195040, {0x63c85b8, 0xc02c599ad0}, 0xc0781e2820, 0xc054fd59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02c599ad0}, 0x660089, {0xc04c614ea0, {0x1743011024887ab4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d21b400, {0x63c85b8, 0xc02c599ad0}, 0xc0b4d7b900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024887ab4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024887ab4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02c599950}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c599950}, {0xc054fd73f8, 0x660089}, 0xc054fd7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02c599950}, {0x634e480}, 0xc0b4d7b700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02c5997d0}, 0xc0b4d7b700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}, 0xc0278e0900, 0xc02e9d0138) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}, 0x3, 0xc02375fd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc054fd7980}}}, {0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}, 0xc0278e0900, 0xc02375fd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02c5997d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c5997d0}, {0x40, 0xc054fd7b70}, 0xc054fd7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02c5997d0}, {0x4e9d7a0, 0xc0b4d7b700}, 0xc0278e0900, 0xc02e9d0138) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02c5997d0}, 0xc043ecb020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04c614b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04c614b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811639 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc025d79540, {0x63c85b8, 0xc020841140}, 0xc04170fab8, 0x1, 0x708018167fd76b, 0x37e11d600, 0xc064c09aa0, 0xc0735e9500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc020841140}, 0x1e0, 0xc033897230, 0x64c09a40, 0x0, 0x660069, 0xc064c09aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc025d79540, {0x63c85b8, 0xc020841140}, 0xc064c09a40, {{{{0x0, 0x0}, {0xc0ada1a000, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc020841140}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc025d3cf50, {0x63c85b8, 0xc020841140}, {0xc01cb545a0, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc025d3cf50, {0x63c85b8, 0xc020841140}, 0xc03e98b2c0, 0xc0338979f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc020841140}, 0x660089, {0xc01cb545a0, {0x1743011067cacab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025dbbc00, {0x63c85b8, 0xc020841140}, 0xc0689b2400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067cacab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020841020}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020841020}, {0xc0338993f8, 0x660089}, 0xc033899400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020841020}, {0x7fe41feec100}, 0xc0689b2200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020840f30}, 0xc0689b2200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}, 0xc086728d00, 0xc04170fa88) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}, 0x3, 0xc0773fdac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc033899980}}}, {0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}, 0xc086728d00, 0xc0773fdac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020840f30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020840f30}, {0x40, 0xc033899b70}, 0xc033899ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020840f30}, {0x4e9d7a0, 0xc0689b2200}, 0xc086728d00, 0xc04170fa88) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020840f30}, 0xc0c45dcf60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc065ff2ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc065ff2ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 216623209 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0511ddb50, 0xed) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc05460aa80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc05460b0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0658ccc00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0658ccc00, {0x63c8510, 0xc0511ddb80}, 0xc05be0fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02f22e3a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0511ddb80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02f22e3a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809948 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc030ef1860}, 0xc074a63cf8, 0x1, 0xc011ec90a8, 0x519bbc, 0xc0613dfc80, 0xc059b49920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc054dc8ba0, {0x63c85b8, 0xc030ef1860}, 0xc0971b3440, 0xc011ec91a8, 0x613dfce0, 0x0, 0x0, 0xc0613dfc80, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc030ef1860}, 0xc0613dfc20, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc045559d40, 0xa8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc030ef1860}, 0x63523b0, 0x1b308ef8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc030ef1860}, {0x0, {0x174301106eb98414, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc030ef1860}, 0xc0479a9ea0, 0xc011ec99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc030ef1860}, 0x660089, {0x0, {0x174301106eb98414, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc030ef1860}, 0xc074606100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106eb98414, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106eb98414, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301106eb98414, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc030ef17a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030ef17a0}, {0xc011ecb3f8, 0x660089}, 0xc011ecb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc030ef17a0}, {0x7fe137ac2d80}, 0xc0b5d8be00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc030ef15f0}, 0xc0b5d8be00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}, 0xc0267a7360, 0xc07289be00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}, 0x3, 0xc05b2d2240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc011ecb980}}}, {0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}, 0xc0267a7360, 0xc05b2d2240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc030ef15f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030ef15f0}, {0x40, 0xc011ecbb70}, 0xc011ecbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc030ef15f0}, {0x4e9d7a0, 0xc0b5d8be00}, 0xc0267a7360, 0xc07289be00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc030ef15f0}, 0xc028b70ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0613dfb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0613dfb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809253 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021ab8000, {0x63c85b8, 0xc0119aa5a0}, 0xc055d21300, 0xc01ea03b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021ab8000, {0x63c85b8, 0xc0119aa5a0}, 0xc055d21300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110638c955a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110638c955a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0119aa450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0119aa450}, {0xc017ecf3f8, 0x660089}, 0xc017ecf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0119aa450}, {0x0}, 0xc055d21100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0119aa3c0}, 0xc055d21100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}, 0xc08a2ada60, 0xc0139fc210) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}, 0x3, 0xc0b877aa40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017ecf980}}}, {0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}, 0xc08a2ada60, 0xc0b877aa40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0119aa3c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0119aa3c0}, {0x40, 0xc017ecfb70}, 0xc017ecfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0119aa3c0}, {0x4e9d7a0, 0xc055d21100}, 0xc08a2ada60, 0xc0139fc210) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0119aa3c0}, 0xc0453b5260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc073795d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc073795d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803849 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc0866a9d10}, 0xc0158b1a70, 0x1, 0xc0798890a8, 0x519bbc, 0xc04d664600, 0xc04e151c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0866a9d10}, 0x1e0, 0xc079889230, 0x4d6645a0, 0x0, 0x660069, 0xc04d664600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc0866a9d10}, 0xc04d6645a0, {{{{0x0, 0x0}, {0xc0756ce6c0, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0866a9d10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc0866a9d10}, {0xc058b7c7e0, {0x174301103285391d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc0866a9d10}, 0xc04bbd19a0, 0xc0798899f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0866a9d10}, 0x660089, {0xc058b7c7e0, {0x174301103285391d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0866a9d10}, 0xc051cf0100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103285391d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103285391d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0866a9c80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0866a9c80}, {0xc07988b3f8, 0x660089}, 0xc07988b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0866a9c80}, {0x7fe3c018d968}, 0xc066a15f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0866a9bf0}, 0xc066a15f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}, 0xc055027a60, 0xc025fdf038) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}, 0x3, 0xc03d2d9140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07988b980}}}, {0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}, 0xc055027a60, 0xc03d2d9140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0866a9bf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0866a9bf0}, {0x40, 0xc07988bb70}, 0xc07988bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0866a9bf0}, {0x4e9d7a0, 0xc066a15f00}, 0xc055027a60, 0xc025fdf038) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0866a9bf0}, 0xc0a976f740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc058b7c5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc058b7c5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810008 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0395e2ea0}, 0xc016471cb0, {0x646cc48, 0xc0395e2f00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0395e2ea0}, 0xedb7a8350, 0xc057603200) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0395e2ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc051603c50}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc016471ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc051603c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812655 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803993 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01edcb400, {0x63c85b8, 0xc0a30112f0}, 0xc092502d00, 0xc039b4a460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc0a30112f0}, 0xc092502d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032291dd2, 0x0, 0x0}, 0xc032a7e6f0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032291dd2, 0x0, 0x0}, 0xc032a7e6f0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0a3011260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a3011260}, {0xc065d1d3f8, 0x660089}, 0xc065d1d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0a3011260}, {0x0}, 0xc092502b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0a30111d0}, 0xc092502b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}, 0xc073f7fc40, 0xc0257e6a98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}, 0x3, 0xc076351ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc065d1d980}}}, {0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}, 0xc073f7fc40, 0xc076351ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0a30111d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a30111d0}, {0x40, 0xc065d1db70}, 0xc065d1dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0a30111d0}, {0x4e9d7a0, 0xc092502b00}, 0xc073f7fc40, 0xc0257e6a98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0a30111d0}, 0xc0a7a54a80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050d4f7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050d4f7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809783 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc011da9800, {0x63c85b8, 0xc087448660}, 0xc057e6bc00, 0xc026033f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc011da9800, {0x63c85b8, 0xc087448660}, 0xc057e6bc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011066ffd8e2, 0x0, 0x0}, 0xc01e9146a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011066ffd8e2, 0x0, 0x0}, 0xc01e9146a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0874485d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0874485d0}, {0xc019ec73f8, 0x660089}, 0xc019ec7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0874485d0}, {0x0}, 0xc057e6b800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc087448540}, 0xc057e6b800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}, 0xc03248a040, 0xc0403c1db8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}, 0x3, 0xc046a15740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc019ec7980}}}, {0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}, 0xc03248a040, 0xc046a15740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc087448540}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc087448540}, {0x40, 0xc019ec7b70}, 0xc019ec7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc087448540}, {0x4e9d7a0, 0xc057e6b800}, 0xc03248a040, 0xc0403c1db8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc087448540}, 0xc06c741920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07d459b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07d459b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809987 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc0594deae0}, 0xc02ecc9c50, 0x1, 0xc0133af0a8, 0x519bbc, 0xc0679c5a40, 0xc07dd36cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0594deae0}, 0x120, 0xc0133af230, 0x679c59e0, 0x0, 0x660069, 0xc0679c5a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc0594deae0}, 0xc0679c59e0, {{{{0x0, 0x0}, {0xc082318b40, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0594deae0}, 0x6352410, 0x29b7bfc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc0594deae0}, {0xc0679c58c0, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc0594deae0}, 0xc0466e05a0, 0xc0133af9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0594deae0}, 0x660089, {0xc0679c58c0, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc0594deae0}, 0xc05feb7f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0594dea50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0594dea50}, {0xc0133b13f8, 0x660089}, 0xc0133b1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0594dea50}, {0x0}, 0xc05feb7d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0594de990}, 0xc05feb7d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}, 0xc032ba5060, 0xc0321fde60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}, 0x3, 0xc05b30ca80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0133b1980}}}, {0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}, 0xc032ba5060, 0xc05b30ca80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0594de990}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0594de990}, {0x40, 0xc0133b1b70}, 0xc0133b1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0594de990}, {0x4e9d7a0, 0xc05feb7d00}, 0xc032ba5060, 0xc0321fde60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0594de990}, 0xc0205776e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c57a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c57a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219152287 [select, 1 minutes]: google.golang.org/grpc.newClientStreamWithParams.func4() google.golang.org/grpc/external/org_golang_google_grpc/stream.go:341 +0x98 created by google.golang.org/grpc.newClientStreamWithParams google.golang.org/grpc/external/org_golang_google_grpc/stream.go:340 +0xb65 goroutine 219651838 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc067673440) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc02dd15550, {0x4d83cc0, 0xc071950d80}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x11b) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc067673200, 0xc072aa0f68, 0xc072aa0f38) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc067673200, {0x4d83cc0, 0xc071950d80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc07aad1f80}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc05707f700}, {0x4d83cc0, 0xc071950d80}, 0x2, {0xc012856840, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc07aad1f80}, {0x4f6406d, 0x7fe457d80878}, {0x4e9d7a0, 0xc05707f700}, {0x4d83cc0, 0xc071950d80}, 0x7fdf1c995eb244d3, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x515f27ba48f8039a, {0x63c85b8, 0xc07aad1f80}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc05707f700}, {0x4d83cc0, 0xc071950d80}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc01caf9100, {0x63c85b8, 0xc07aad1f80}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc01caf9100}}, {0x63c85b8, 0xc07aad1f80}, 0xc05707f700, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1c, 0x1c, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc05b5232c0, {0x63c85b8, 0xc07aad1f20}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc05b5232c0, {0x63c85b8, 0xc07aad1f20}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc0b255f270, {0x63c85b8, 0xc07aad1f20}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0a26cee00, {0x63c85b8, 0xc07aad1f20}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc04e572be0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc03e391980) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc037f3b560) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).next(0xc0640998c0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:374 +0x7d github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc0640998c0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:189 +0x32c github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc03e391a00) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*isNullSelOp).Next(0xc07aad1e60) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/is_null_ops.eg.go:226 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0dc02c900) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc04ba74160, {0x63c8510, 0xc03e391940}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc09270eaf0, {0x63c8510, 0xc03e391940}, 0xc063fe7970) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc03e391540}, 0xc08cbc8620, 0xc063fe7970, 0xc071950b80, 0xc0bef3bc00, 0xc0e2f295e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc03e391540}, 0xc0e2f292f0, 0xc08cbc8620, 0xc03bfa66c0, {{0x63ca340, 0xc071950b00}, 0x0}, 0xc0bef3bc00) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0e2f28c00, {0x63c8510, 0xc03e391540}, 0xc0e2f292f0, 0xc03e391540, {0x7fe42b790cf0, 0xc03bfa66c0}, 0x0, 0xc037f3b378) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0e2f28c00, {0x63c8510, 0xc03e391540}, 0xc0e2f292f0, {0x7fe42b790cf0, 0xc03bfa66c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0e2f28c00, {0x63c85b8, 0xc05bcf75f0}, {{0x63f8d78, 0xc064c5ff90}, {0xc0776d4b6d, 0xb4}, 0x4, 0x1}, 0xc060849b00, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0e2f28c00, {0x63c85b8, 0xc05bcf75f0}, {{0x63f8d78, 0xc064c5ff90}, {0xc0776d4b6d, 0xb4}, 0x4, 0x1}, 0xc060849b00, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0e2f28c00, {0x63c85b8, 0xc05bcf75f0}, {0xc060849b00, {0xc03e391500, 0x4, 0x4}, {0xc026a9afa8, 0xb, 0xb}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0776d4cae, 0x0}, 0x0, {0x27dd5fe7, 0xedb7a8349, 0x0}, 0x1}, 0xc0e2f28c00, 0xc072aa7980, 0xc072aa7970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0e2f28c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0e2f28c00, {0x63c8510, 0xc016c9af40}, 0xc056cb1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03b5f58a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc016c9af40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03b5f58a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810803 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc073e82c90}, 0xc03a7801f8, 0x1, 0xc024dcf0a8, 0x519bbc, 0xc05fdef920, 0xc0642b1380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc073e82c90}, 0x120, 0xc024dcf230, 0x5fdef8c0, 0x0, 0x660069, 0xc05fdef920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc073e82c90}, 0xc05fdef8c0, {{{{0x0, 0x0}, {0xc0bc5b9200, 0xfd}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073e82c90}, 0x6352410, 0x1e651558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc073e82c90}, {0xc05fdef7a0, {0x1743011063852431, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc073e82c90}, 0xc0767670e0, 0xc024dcf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073e82c90}, 0x660089, {0xc05fdef7a0, {0x1743011063852431, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc073e82c90}, 0xc08adce600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063852431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063852431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073e82c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073e82c00}, {0xc024dd13f8, 0x660089}, 0xc024dd1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073e82c00}, {0xc0015221e0}, 0xc08adce300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073e82b70}, 0xc08adce300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}, 0xc013698060, 0xc046e055f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}, 0x3, 0xc0408a4280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc024dd1980}}}, {0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}, 0xc013698060, 0xc0408a4280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073e82b70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073e82b70}, {0x40, 0xc024dd1b08}, 0xc024dd1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073e82b70}, {0x4e9d7a0, 0xc08adce300}, 0xc013698060, 0xc046e055f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073e82b70}, 0xc05f453c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc02032efc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc02032efc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219781832 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea27d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05ba90480, 0xc05560a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05ba90480, {0xc05560a000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05ba90480, {0xc05560a000, 0x735, 0xc050069680}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc020773178, {0xc05560a000, 0x7eda79, 0xc05560a005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc09659dc20, {0xc05560a000, 0xc071814c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc08a887f38, {0xc05560a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0534d3078, {0x631b900, 0xc08a887f38}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0534d2e00, {0x7fe42ab59310, 0xc09659dc20}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0534d2e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0534d2e00, {0xc0002f5000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc04318f8a8, {0xc0002f5000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0718d4b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0718d4b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0718d50e0, {0x634aac0, 0xc0718d4b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0718d4a80, 0xc08c6a5fb0, 0xc09520b42b, 0xc09520b42d, {0x635bec0, 0xc05a273440}, 0xc000d6ae00, 0xc04efc2bd8, 0xc09520b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0718d4a80, {0x63c85b8, 0xc011392900}, 0xc08c6a5fa0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06228c038, 0x8}}, 0x0, 0xc011392810, 0xc0113928d0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc011392900}, {0x643d810, 0xc09659dc20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06e988de0}, {0x643d810, 0xc09659dc20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06e988de0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806149 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc070e0ea80}, 0xc04a416800, 0xc02044f400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc070e0ea80}, 0xc04a416800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104be59ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104be59ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070e0e9f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070e0e9f0}, {0xc0073793f8, 0x660089}, 0xc007379400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070e0e9f0}, {0x39342d363938322d}, 0xc04a416600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070e0e960}, 0xc04a416600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}, 0xc040c359e0, 0xc08a656f48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}, 0x3, 0xc067947300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007379980}}}, {0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}, 0xc040c359e0, 0xc067947300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070e0e960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070e0e960}, {0x40, 0xc007379b70}, 0xc007379ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070e0e960}, {0x4e9d7a0, 0xc04a416600}, 0xc040c359e0, 0xc08a656f48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070e0e960}, 0xc06d8ba4e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc035f0cc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc035f0cc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812265 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc053ec6d20}, 0xc04c38e7e0, 0x1, 0xc037c3d0a8, 0x519bbc, 0xc013d7b260, 0xc02f276600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc053ec6d20}, 0x120, 0xc037c3d230, 0x13d7b200, 0x0, 0x660069, 0xc013d7b260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc053ec6d20}, 0xc013d7b200, {{{{0x0, 0x0}, {0xc06f5fd200, 0x34}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc053ec6d20}, 0x6352410, 0x65f8a688) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc053ec6d20}, {0xc013d7b0e0, {0x1743011077a7adfe, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc053ec6d20}, 0xc055057ae0, 0xc037c3d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc053ec6d20}, 0x660089, {0xc013d7b0e0, {0x1743011077a7adfe, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc053ec6d20}, 0xc04ecf6100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077a7adfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077a7adfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053ec6960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053ec6960}, {0xc037c3f3f8, 0x660089}, 0xc037c3f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053ec6960}, {0x7fde180bccc0}, 0xc04983df00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053ec68d0}, 0xc04983df00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}, 0xc03294fe20, 0xc04c38f938) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}, 0x3, 0xc0638c5c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc037c3f980}}}, {0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}, 0xc03294fe20, 0xc0638c5c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc053ec68d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053ec68d0}, {0x40, 0xc037c3fb70}, 0xc037c3fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc053ec68d0}, {0x4e9d7a0, 0xc04983df00}, 0xc03294fe20, 0xc04c38f938) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc053ec68d0}, 0xc065334e40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc013d7aea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc013d7aea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812656 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802544 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e114640, {0x63c85b8, 0xc086bbac30}, 0xc04abc1f98, 0x1, 0xc05b7130a8, 0x519bbc, 0xc077282f00, 0xc04dcf8ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc086bbac30}, 0x120, 0xc05b713230, 0x77282ea0, 0x0, 0x660069, 0xc077282f00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e114640, {0x63c85b8, 0xc086bbac30}, 0xc077282ea0, {{{{0x0, 0x0}, {0xc089a88b40, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc086bbac30}, 0x6352230, 0x561afb00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dd810e0, {0x63c85b8, 0xc086bbac30}, {0xc077282d80, {0x17430110246c4129, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dd810e0, {0x63c85b8, 0xc086bbac30}, 0xc08f98a8c0, 0xc05b7139f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc086bbac30}, 0x660089, {0xc077282d80, {0x17430110246c4129, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e147000, {0x63c85b8, 0xc086bbac30}, 0xc0a38f8800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110246c4129, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110246c4129, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086bbaab0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086bbaab0}, {0xc05b7153f8, 0x660089}, 0xc05b715400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086bbaab0}, {0xc0015221e0}, 0xc0a38f8600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086bbaa20}, 0xc0a38f8600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}, 0xc0270b5d60, 0xc042d4a8d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}, 0x3, 0xc05511a5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05b715980}}}, {0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}, 0xc0270b5d60, 0xc05511a5c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086bbaa20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086bbaa20}, {0x40, 0xc05b715b70}, 0xc05b715ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086bbaa20}, {0x4e9d7a0, 0xc0a38f8600}, 0xc0270b5d60, 0xc042d4a8d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086bbaa20}, 0xc030802360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc077282c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc077282c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219770009 [semacquire]: sync.runtime_SemacquireMutex(0xc091150e98, 0x66, 0xc00074af80) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0x2bf65a7, 0x15c8d69, 0x7fd7c4d15640) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x1199135, 0xc091150f08, 0xc091150f88, 0x11866c6) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x4d866e, 0x9185600}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).init(0xc02331c480, 0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1229 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc00e0b2580, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2157 +0xdb github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a528c0, 0xc091151228, 0xc0bd8448e8, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc0bd844a00, 0x0, 0xc0bd844a00) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0bd844898, 0xc0734106e0, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0bd844898, {0xc02db19200, 0x72, 0x74}, {0xc021c13980, 0x72, 0x72}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0bd8431c0, {0xc021c13980, 0x464ed60, 0x1}, 0xc0911515c8, 0xc8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0bd8431c0, {0xc02db19200, 0x72, 0xc00e526c80}, {0xc021c13980, 0xc00e526c80, 0x71}, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0bd842f00, {0xc021c13980, 0x72, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04f9daea8, {{0xc00e526c80, 0x71, 0x140}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc08dbb2100, {{0xc04fec47e0, 0x6a, 0xe0}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0529aac00, {0x63c85b8, 0xc059b06e40}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc059b06e40}, {0x64a6560, 0xc08dbb2100}, {0xc04fec47e0, 0x0, 0xe0}, {0x174301109eeab8f5, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc059b06e40}, {0x6466ed8, 0xc04f9dac00}, {0xc04fec47e0, 0x6a, 0xe0}, {0x174301109eeab8f5, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0672e9b48}, {{0x174301109eeab8f5, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109eeab8f5, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03e5b2300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bcb81df5, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e781800, {0x63c85b8, 0xc059b06e40}, 0xc03e5b2300, 0xc0cdadcf00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e781800, {0x63c85b8, 0xc059b06e40}, 0xc03e5b2300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109eeab8f5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109eeab8f5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059b06d50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059b06d50}, {0xc091157498, 0x660089}, 0xc0911574a0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059b06d50}, {0xc00102e4e0}, 0xc03e5b2200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc059b06cf0}, 0xc03e5b2200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc04a9c6960, {0x63c85b8, 0xc059b06c00}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc04a9c6960, {0x63c85b8, 0xc059b06c00}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc01a21a8c0, {0x63c85b8, 0xc059b06c00}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc03e336380, {0x63c85b8, 0xc059b06c00}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc04a2975e0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0492825c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc08a576000) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc070f300b0, {0x63c8510, 0xc049282580}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0418fb310, {0x63c8510, 0xc049282580}, 0xc041e1f760) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc049282480}, 0xc09f7cf7a0, 0xc041e1f760, 0xc062282880, 0xc08d4bce00, 0xc0a14eefe8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc049282480}, 0xc0a14eecf0, 0xc09f7cf7a0, 0xc04d6bf140, {{0x63caae8, 0xc070056460}, 0x0}, 0xc08d4bce00) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0a14ee600, {0x63c8510, 0xc049282480}, 0xc0a14eecf0, 0xc049282480, {0x7fe42b790cf0, 0xc04d6bf140}, 0x0, 0xc078dbc4d8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0a14ee600, {0x63c8510, 0xc049282480}, 0xc0a14eecf0, {0x7fe42b790cf0, 0xc04d6bf140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0a14ee600, {0x63c85b8, 0xc059b06660}, {{0x63f8d78, 0xc0464e0c30}, {0xc04ec78370, 0x65}, 0x2, 0x1}, 0xc0512ffb00, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0a14ee600, {0x63c85b8, 0xc059b06660}, {{0x63f8d78, 0xc0464e0c30}, {0xc04ec78370, 0x65}, 0x2, 0x1}, 0xc0512ffb00, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0a14ee600, {0x63c85b8, 0xc059b06660}, {0xc0512ffb00, {0xc01b611b40, 0x2, 0x2}, {0xc04cfc0960, 0x4, 0x4}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc00fed9fb6, 0x0}, 0x0, {0xc789075, 0xedb7a8351, 0x0}, 0x1}, 0xc0a14ee600, 0xc09115d980, 0xc09115d970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0a14ee600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0a14ee600, {0x63c8510, 0xc0119c4d80}, 0xc08e159dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02c19cb20) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0119c4d80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02c19cb20) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812544 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc04f5ed8c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0d0462e00, 0xc03fd17168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04f5ed8c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0d0462e00, {0xc04c9a36e0, 0x29, 0x60}, {0xc00d09ba40, 0x29, 0x6e}, 0x0, 0x76) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092f87538, {0xc04c9a36e0, 0x1, 0x6e}, {0xc00d09ba40, 0xc092f86500, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092f87538, {0xc04c9a36e0, 0x29, 0x60}, {0xc00d09ba40, 0x29, 0x6e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092f867c0, {0xc00d09ba40, 0x0, 0xc03fd174e0}, 0x121ea30, 0x60) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092f867c0, {0xc04c9a36e0, 0x29, 0x6e}, {0xc00d09ba40, 0x28, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092f86500, {0xc00d09ba40, 0x29, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04c438588, {{0xc03a992960, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc058ad3300, {{0xc03a992960, 0x28, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc025731800, {0x63c85b8, 0xc04bf65830}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04bf65830}, {0x64a6560, 0xc058ad3300}, {0xc03a992960, 0x0, 0x30}, {0x17430110883674a2, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04bf65830}, {0x6466ed8, 0xc04c438580}, {0xc03a992960, 0x28, 0x30}, {0x17430110883674a2, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0232389d8}, {{0x17430110883674a2, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110883674a2, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04aa85e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a603d9a2, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0312b0400, {0x63c85b8, 0xc04bf65830}, 0xc04aa85e00, 0xc085728140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0312b0400, {0x63c85b8, 0xc04bf65830}, 0xc04aa85e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110883674a2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110883674a2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bf657a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf657a0}, {0xc03fd1d3f8, 0x660089}, 0xc03fd1d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bf657a0}, {0x22}, 0xc04aa85c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bf656e0}, 0xc04aa85c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}, 0xc0ccf7f8e0, 0xc0381618f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}, 0x3, 0xc071e54280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03fd1d980}}}, {0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}, 0xc0ccf7f8e0, 0xc071e54280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bf656e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf656e0}, {0x40, 0xc03fd1db70}, 0xc03fd1dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bf656e0}, {0x4e9d7a0, 0xc04aa85c00}, 0xc0ccf7f8e0, 0xc0381618f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bf656e0}, 0xc05c2fb4a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806565 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc08cc21800}, 0xc022c5e2e8, 0x1, 0xc0053fb0a8, 0x519bbc, 0xc063afb140, 0xc08fb9eba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08cc21800}, 0x120, 0xc0053fb230, 0x63afb0e0, 0x0, 0x660069, 0xc063afb140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc08cc21800}, 0xc063afb0e0, {{{{0x0, 0x0}, {0xc07b98c480, 0x5b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08cc21800}, 0x6352410, 0x45bf51e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc08cc21800}, {0xc063afafc0, {0x174301104e71cee3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc08cc21800}, 0xc072bc5ae0, 0xc0053fb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08cc21800}, 0x660089, {0xc063afafc0, {0x174301104e71cee3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc08cc21800}, 0xc08bbfc300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e71cee3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e71cee3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08cc21770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08cc21770}, {0xc0053fd3f8, 0x660089}, 0xc0053fd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08cc21770}, {0xc0053fd490}, 0xc08bbfc100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08cc216e0}, 0xc08bbfc100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}, 0xc08dea51c0, 0xc0652ca3d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}, 0x3, 0xc087cbb980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0053fd980}}}, {0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}, 0xc08dea51c0, 0xc087cbb980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08cc216e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08cc216e0}, {0x40, 0xc0053fdb70}, 0xc0053fdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08cc216e0}, {0x4e9d7a0, 0xc08bbfc100}, 0xc08dea51c0, 0xc0652ca3d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08cc216e0}, 0xc0751f1380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc063afaea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc063afaea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219617697 [IO wait]: internal/poll.runtime_pollWait(0x7fe41fe993c8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04e335480, 0xc02bdc3000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04e335480, {0xc02bdc3000, 0x2b8d, 0x2b8d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04e335480, {0xc02bdc3000, 0x2b8d, 0xc0d100b2c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc07818f7d8, {0xc02bdc3000, 0x7eda79, 0xc02bdc3005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc049ebade0, {0xc02bdc3000, 0x0, 0x2b8d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0342fce58, {0xc02bdc3000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc086bd2278, {0x631b900, 0xc0342fce58}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc086bd2000, {0x7fe42ab59310, 0xc049ebade0}, 0x2b8d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc086bd2000, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc086bd2000, {0xc04af90000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc007a4bd88, {0xc04af90000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc064cfeb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc064cfeb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc064cff0e0, {0x634aac0, 0xc064cfeb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc064cfea80, 0xc0121f3ee0, 0xc075c7f42b, 0xc075c7f42d, {0x635bec0, 0xc033c04d80}, 0xc000d6ae00, 0xc05a36d534, 0xc075c7f429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc064cfea80, {0x63c85b8, 0xc02fe51650}, 0xc0121f3ed0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc076f7b009, 0x8}}, 0x0, 0xc02fe515f0, 0xc02fe51620, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02fe51650}, {0x643d810, 0xc049ebade0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc077517bc0}, {0x643d810, 0xc049ebade0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc077517bc0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812609 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc053000380, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc0415af198, 0xc09272a438, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc09272a550, 0x0, 0xc09272a550) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc09272a3e8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09272a3e8, {0xc03a31e0c0, 0x18, 0x51}, {0xc022036930, 0x18, 0x6e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0927291c0, {0xc022036930, 0x0, 0xc0415af4e0}, 0x121ea30, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0927291c0, {0xc03a31e0c0, 0x18, 0x6e}, {0xc022036930, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092728f00, {0xc022036930, 0x18, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06f98eb08, {{0xc08467a900, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07b49c000, {{0xc08467a900, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc073bf5000, {0x63c85b8, 0xc04f63b650}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04f63b650}, {0x64a6560, 0xc07b49c000}, {0xc08467a900, 0x0, 0x18}, {0x174301109470708a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04f63b650}, {0x6466ed8, 0xc06f98eb00}, {0xc08467a900, 0x17, 0x18}, {0x174301109470708a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc079b5a600}, {{0x174301109470708a, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109470708a, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0455a3300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b23dd58a, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc04f63b650}, 0xc0455a3300, 0xc06f938320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc04f63b650}, 0xc0455a3300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109470708a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109470708a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63b5c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b5c0}, {0xc0415b53f8, 0x660089}, 0xc0415b5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63b5c0}, {0xc0015221e0}, 0xc0455a3100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63b530}, 0xc0455a3100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}, 0xc03ebdd520, 0xc031cfb050) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}, 0x3, 0xc033d33980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0415b5980}}}, {0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}, 0xc03ebdd520, 0xc033d33980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63b530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b530}, {0x40, 0xc0415b5b70}, 0xc0415b5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63b530}, {0x4e9d7a0, 0xc0455a3100}, 0xc03ebdd520, 0xc031cfb050) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63b530}, 0xc039f0a780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e065c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e065c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802675 [runnable]: sync.runtime_SemacquireMutex(0xc094a8e690, 0x8f, 0x9bb12c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc094a8e538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc094a8e560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0601b3200, {0xffffffffffffff15, 0xc094a8e6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0601b3200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0601b3200, {0xc02447cf50, 0x32, 0x50}, {0xc01ef7d280, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092733998, {0xc02447cf50, 0x32, 0x50}, {0xc01ef7d280, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0927322c0, {0xc01ef7d280, 0x79, 0x50}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0927322c0, {0xc02447cf50, 0x79, 0xc01ef7d280}, {0xc01ef7d280, 0xc092017380, 0x31}, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092732000, {0xc01ef7d280, 0x32, 0x79}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc05a2daea8, {{0xc092017380, 0x31, 0xd0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0745af700, {{0xc0650a4f30, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc053e26400, {0x63c85b8, 0xc049dd4900}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc049dd4900}, {0x64a6560, 0xc0745af700}, {0xc0650a4f30, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc049dd4900}, {0x6466ed8, 0xc05a2dac00}, {0xc0650a4f30, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc042f81698}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0b5830300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4c400, {0x63c85b8, 0xc049dd4900}, 0xc0b5830300, 0xc0ab4d6960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4c400, {0x63c85b8, 0xc049dd4900}, 0xc0b5830300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc049dd4870}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc049dd4870}, {0xc0ad8388f0, 0x660089}, 0xc0ad8388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc049dd4870}, {0xc00102e4e0}, 0xc0b5830200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc049dd4810}, 0xc0b5830200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220cf0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804879 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc048382480}, 0xc0483f6ee8, 0x1, 0x708017d894be1a, 0x37e11d600, 0xc06f85cde0, 0xc03aac5da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc048382480}, 0x120, 0xc07b325230, 0x6f85cd80, 0xc0681391e0, 0x18, 0xc06f85cde0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc048382480}, 0xc06f85cd80, {{{{0x0, 0x0}, {0xc055a9b200, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc048382480}, 0x6352410, 0x1b25d2d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc048382480}, {0xc06f85cc60, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc048382480}, 0xc05f1b30e0, 0xc07b3259f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc048382480}, 0x660089, {0xc06f85cc60, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc048382480}, 0xc0878b1d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0483823f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0483823f0}, {0xc07b3273f8, 0x660089}, 0xc07b327400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0483823f0}, {0x7fe429365788}, 0xc0878b1b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048382300}, 0xc0878b1b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}, 0xc0235c52a0, 0xc0483f6eb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}, 0x3, 0xc01ae26640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b327980}}}, {0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}, 0xc0235c52a0, 0xc01ae26640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048382300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048382300}, {0x40, 0xc07b327b70}, 0xc07b327ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}, 0x203006, 0x203006) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048382300}, {0x4e9d7a0, 0xc0878b1b00}, 0xc0235c52a0, 0xc0483f6eb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048382300}, 0xc027dfa0c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85c7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85c7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804970 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03005e780, {0x63c85b8, 0xc01b30c450}, 0xc0299e3f50, 0x1, 0xc0591230a8, 0x519bbc, 0xc06eaf36e0, 0xc04e285260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01b30c450}, 0x120, 0xc059123230, 0x6eaf3680, 0xc068139380, 0x18, 0xc06eaf36e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03005e780, {0x63c85b8, 0xc01b30c450}, 0xc06eaf3680, {{{{0x0, 0x0}, {0xc075903560, 0xd}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01b30c450}, 0x6352410, 0x421b18b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030007040, {0x63c85b8, 0xc01b30c450}, {0xc06eaf3560, {0x1743011030e52d02, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030007040, {0x63c85b8, 0xc01b30c450}, 0xc050935860, 0xc0591239f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01b30c450}, 0x660089, {0xc06eaf3560, {0x1743011030e52d02, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030065c00, {0x63c85b8, 0xc01b30c450}, 0xc01aa11a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011030e52d02, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011030e52d02, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b30c2a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b30c2a0}, {0xc0591253f8, 0x660089}, 0xc059125400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b30c2a0}, {0x7fe4292eba20}, 0xc01aa11700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b30c210}, 0xc01aa11700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}, 0xc013bebe80, 0xc05fe66318) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}, 0x3, 0xc06f13db40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc059125980}}}, {0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}, 0xc013bebe80, 0xc06f13db40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01b30c210}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b30c210}, {0x40, 0xc059125b70}, 0xc059125ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01b30c210}, {0x4e9d7a0, 0xc01aa11700}, 0xc013bebe80, 0xc05fe66318) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01b30c210}, 0xc02a4cc960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06eaf3440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06eaf3440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801381 [runnable]: sync.runtime_SemacquireMutex(0xc09749c690, 0x8f, 0x7fe3f6c18bc0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc09749c538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc09749c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04b97e480, {0xffffffffffffff15, 0xc09749c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc04b97e480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04b97e480, {0xc04f884c40, 0x32, 0x6b}, {0xc030cc1480, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092849e98, {0xc04f884c40, 0x32, 0x6b}, {0xc030cc1480, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0928487c0, {0xc030cc1480, 0x72, 0x6b}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0928487c0, {0xc04f884c40, 0x72, 0xc030cc1480}, {0xc030cc1480, 0xc057f81680, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092848500, {0xc030cc1480, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0593e6da8, {{0xc057f81680, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04ec1de00, {{0xc0674de300, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc044dc2800, {0x63c85b8, 0xc0370b9e30}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0370b9e30}, {0x64a6560, 0xc04ec1de00}, {0xc0674de300, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0370b9e30}, {0x6466ed8, 0xc0593e6b00}, {0xc0674de300, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01675e6c0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0af507d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c61400, {0x63c85b8, 0xc0370b9e30}, 0xc0af507d00, 0xc05aafc460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c61400, {0x63c85b8, 0xc0370b9e30}, 0xc0af507d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0370b9d40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0370b9d40}, {0xc0935988f0, 0x660089}, 0xc0935988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0370b9d40}, {0xc00102e4e0}, 0xc0af507b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0370b9ce0}, 0xc0af507b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc017f71e00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805408 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0310be400, {0x63c85b8, 0xc03ffa94d0}, 0xc051531000, 0xc045d46c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310be400, {0x63c85b8, 0xc03ffa94d0}, 0xc051531000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011044d2d939, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011044d2d939, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03ffa9440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ffa9440}, {0xc0941693f8, 0x660089}, 0xc094169400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03ffa9440}, {0x7fe422b97658}, 0xc051530e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03ffa93b0}, 0xc051530e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}, 0xc03a017c80, 0xc031fad098) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}, 0x3, 0xc0661b54c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc094169980}}}, {0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}, 0xc03a017c80, 0xc0661b54c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03ffa93b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ffa93b0}, {0x40, 0xc094169b70}, 0xc094169ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}, 0x203019, 0x203019) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03ffa93b0}, {0x4e9d7a0, 0xc051530e00}, 0xc03a017c80, 0xc031fad098) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03ffa93b0}, 0xc0534a6300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05fcb6240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05fcb6240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219708170 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1530, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06f017080, 0xc0076c1800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06f017080, {0xc0076c1800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06f017080, {0xc0076c1800, 0x735, 0xc08ac33e00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03eaa0e58, {0xc0076c1800, 0x7eda79, 0xc0076c1805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0418da120, {0xc0076c1800, 0x35318b4, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc026437818, {0xc0076c1800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0483fd078, {0x631b900, 0xc026437818}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0483fce00, {0x7fe42ab59310, 0xc0418da120}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0483fce00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0483fce00, {0xc01988d000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc00fe3be18, {0xc01988d000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05ff3a088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05ff3a088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05ff3a660, {0x634aac0, 0xc05ff3a088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05ff3a000, 0xc034b44710, 0xc07d4c342b, 0xc07d4c342d, {0x635bec0, 0xc0858185a0}, 0xc000d6ae00, 0xc00d4f7c48, 0xc07d4c3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05ff3a000, {0x63c85b8, 0xc074147260}, 0xc034b44700, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc038817038, 0x8}}, 0x0, 0xc0741471a0, 0xc0741471d0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc074147260}, {0x643d810, 0xc0418da120}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04413fb60}, {0x643d810, 0xc0418da120}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04413fb60}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812742 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803426 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc03ccb61b0}, 0xc01dbf9650, 0x1, 0xc0761d70a8, 0x519bbc, 0xc036fcf260, 0xc03593db60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03ccb61b0}, 0x120, 0xc0761d7230, 0x36fcf200, 0x0, 0x660069, 0xc036fcf260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc03ccb61b0}, 0xc036fcf200, {{{{0x0, 0x0}, {0xc07805c000, 0x16}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03ccb61b0}, 0x6352410, 0x49a43788) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc03ccb61b0}, {0xc036fcf0e0, {0x174301102bed681e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc03ccb61b0}, 0xc01450a640, 0xc0761d79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03ccb61b0}, 0x660089, {0xc036fcf0e0, {0x174301102bed681e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc03ccb61b0}, 0xc04f236000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bed681e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bed681e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03ccb6120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ccb6120}, {0xc0761d93f8, 0x660089}, 0xc0761d9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03ccb6120}, {0x3}, 0xc06776de00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067fd5f80}, 0xc06776de00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}, 0xc0433d24c0, 0xc037694528) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}, 0x3, 0xc0549b05c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0761d9980}}}, {0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}, 0xc0433d24c0, 0xc0549b05c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc067fd5f80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067fd5f80}, {0x40, 0xc0761d9b70}, 0xc0761d9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc067fd5f80}, {0x4e9d7a0, 0xc06776de00}, 0xc0433d24c0, 0xc037694528) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc067fd5f80}, 0xc01bbc6960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc036fcefc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc036fcefc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812424 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ebc7a40, {0x63c85b8, 0xc04f5697d0}, 0xc044339d10, 0x1, 0xc03bb0b6d8, 0x519bbc, 0xc01d608540, 0xc06b98a540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc057368c60, {0x63c85b8, 0xc04f5697d0}, 0xc01d608540, 0xc03bb0b7d8, 0x31, 0x40, 0x0, 0xc01d608540, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ebc7a40, {0x63c85b8, 0xc04f5697d0}, 0xc01d608480, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0468c4c60, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f5697d0}, 0x6352260, 0x204d2eb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f582780, {0x63c85b8, 0xc04f5697d0}, {0xc01d608360, {0x17430110567df5e4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f582780, {0x63c85b8, 0xc04f5697d0}, 0xc02e903b80, 0xc03bb0c020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f5697d0}, 0x660089, {0xc01d608360, {0x17430110567df5e4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f5ac400, {0x63c85b8, 0xc04f5697d0}, 0xc070375e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110567df5e4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110567df5e4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f569740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f569740}, {0xc07d93fa28, 0x660089}, 0xc07d93fa30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f569740}, {0xc00102e4e0}, 0xc070375d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f5696e0}, 0xc070375d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc04fc2ecd0, {0x63c85b8, 0xc04f569650}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc04fc2ecd0, {0x63c85b8, 0xc04f569650}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc04f569650}, {0x4f67ad2, 0x10beec0}, 0xc02790dfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219793021 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0654c50e0}, 0xc035840af8, {0x646cc48, 0xc0654c5140}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0654c50e0}, 0xedb7a834e, 0xc07649c900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0654c50e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0206b34d0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc046b76ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0206b34d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805168 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc04cd0cab0}, 0xc0ab174200, 0xc05f537c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc04cd0cab0}, 0xc0ab174200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011041c6d918, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011041c6d918, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04cd0ca20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cd0ca20}, {0xc07c0cb3f8, 0x660089}, 0xc07c0cb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04cd0ca20}, {0xc0015221e0}, 0xc0ab174000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04cd0c990}, 0xc0ab174000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}, 0xc051693960, 0xc025da1980) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}, 0x3, 0xc0132585c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07c0cb980}}}, {0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}, 0xc051693960, 0xc0132585c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04cd0c990}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04cd0c990}, {0x40, 0xc07c0cbb70}, 0xc07c0cbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}, 0x203004, 0x203004) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04cd0c990}, {0x4e9d7a0, 0xc0ab174000}, 0xc051693960, 0xc025da1980) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04cd0c990}, 0xc0183b8c60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05a937560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05a937560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811053 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab97400, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc066de4f00, 0xc00e4be800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc066de4f00, {0xc00e4be800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc066de4f00, {0xc00e4be800, 0x735, 0xc03e43c640}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03c750430, {0xc00e4be800, 0x7eda79, 0xc00e4be805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04abe21e0, {0xc00e4be800, 0xc082bdac60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03b6458d8, {0xc00e4be800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0d05e25f8, {0x631b900, 0xc03b6458d8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0d05e2380, {0x7fe42ab59310, 0xc04abe21e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0d05e2380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0d05e2380, {0xc0a3383000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc072280a80, {0xc0a3383000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04e950b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04e950b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04e9510e0, {0x634aac0, 0xc04e950b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04e950a80, 0xc0204959f0, 0xc082bdb42b, 0xc082bdb42d, {0x635bec0, 0xc03ef1f0e0}, 0xc000d6ae00, 0xc0247f6038, 0xc082bdb429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04e950a80, {0x63c85b8, 0xc0284ac2d0}, 0xc0204959e0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc09f682009, 0x8}}, 0x0, 0xc0284ac270, 0xc0284ac2a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0284ac2d0}, {0x643d810, 0xc04abe21e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc08d942b40}, {0x643d810, 0xc04abe21e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc08d942b40}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812648 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801373 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02a26d000, {0x63c85b8, 0xc0488e3e00}, 0xc065171200, 0xc0601d9040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a26d000, {0x63c85b8, 0xc0488e3e00}, 0xc065171200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c7da084, 0x0, 0x0}, 0xc00d974560, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c7da084, 0x0, 0x0}, 0xc00d974560, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0488e3d70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0488e3d70}, {0xc002c0f3f8, 0x660089}, 0xc002c0f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0488e3d70}, {0x7fe3e0f741a8}, 0xc065171000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0488e3c80}, 0xc065171000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}, 0xc060420c20, 0xc05b165db8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}, 0x3, 0xc08a35fe40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002c0f980}}}, {0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}, 0xc060420c20, 0xc08a35fe40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0488e3c80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0488e3c80}, {0x40, 0xc002c0fb70}, 0xc002c0fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0488e3c80}, {0x4e9d7a0, 0xc065171000}, 0xc060420c20, 0xc05b165db8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0488e3c80}, 0xc048eb42a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ac167440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ac167440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803272 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc07503c600}, 0xc089ff36b0, 0x1, 0xc06ce7dff0, 0x519bbc, 0xc0328e4ba0, 0xc019d102a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07503c600}, 0x120, 0xc06ce7e178, 0x328e4b40, 0x0, 0x660069, 0xc0328e4ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc07503c600}, 0xc0328e4b40, {{{{0x0, 0x0}, {0xc07cbafb00, 0xd4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07503c600}, 0x6352410, 0x3ad19fe0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc07503c600}, {0xc0328e4a20, {0x1743011019292be5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc07503c600}, 0xc05b113180, 0xc08da96938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07503c600}, 0x660089, {0xc0328e4a20, {0x1743011019292be5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc07503c600}, 0xc0262b3a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07503c570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07503c570}, {0xc08da98340, 0x660089}, 0xc08da98348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07503c570}, {0xc00102e4e0}, 0xc0262b3900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07503c510}, 0xc0262b3900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc05934be30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800591 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0184f7c00, {0x63c85b8, 0xc058442f30}, 0xc04be37900, 0xc0496fc1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0184f7c00, {0x63c85b8, 0xc058442f30}, 0xc04be37900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101615db41, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101615db41, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc058442ea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc058442ea0}, {0xc087fa67f8, 0x660089}, 0xc087fa6800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc058442ea0}, {0xc00102e4e0}, 0xc04be37800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc058442e40}, 0xc04be37800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc00d45f340, {0x63c85b8, 0xc058442cf0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc08fe6e000, {0x63c85b8, 0xc058442cf0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc08fe6e000, {0x63c8510, 0xc074f07380}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc08fe6e000, {0x63c8510, 0xc074f07380}, {0x63f7db8, 0x9bf6f48}, 0xc087fab088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc08fe6e000, {0x63c85b8, 0xc04bd09bc0}, {{0x63f7db8, 0x9bf6f48}, {0xc05f5bc3b5, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc08fe6e000, {0x63c85b8, 0xc04bd09bc0}, {{0x63f7db8, 0x9bf6f48}, {0xc05f5bc3b5, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc05f5bc3b5, 0x6}, 0x0, 0x0}, {0x387cef67, 0xedb7a834e, 0x0}, {0x387cf841, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08fe6e000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08fe6e000, {0x63c8510, 0xc06b93ab40}, 0xc076117dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0460543a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06b93ab40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0460543a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809489 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc0200864b0}, 0xc087c6afd8, 0x1, 0xc00e5650a8, 0x519bbc, 0xc087816600, 0xc06da42600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0200864b0}, 0x120, 0xc00e565230, 0x878165a0, 0x0, 0x660069, 0xc087816600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc0200864b0}, 0xc0878165a0, {{{{0x0, 0x0}, {0xc061565e60, 0xf}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0200864b0}, 0x6352410, 0x46e77870) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc0200864b0}, {0xc087816480, {0x174301106409cf95, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc0200864b0}, 0xc070e26c80, 0xc00e5659f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0200864b0}, 0x660089, {0xc087816480, {0x174301106409cf95, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc0200864b0}, 0xc0933c3800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106409cf95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106409cf95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020086030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020086030}, {0xc00e5673f8, 0x660089}, 0xc00e567400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020086030}, {0x7fe3f72ef1a0}, 0xc0933c3600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048e8ff50}, 0xc0933c3600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}, 0xc047483c60, 0xc0c884dc80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}, 0x3, 0xc01b30fec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e567980}}}, {0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}, 0xc047483c60, 0xc01b30fec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048e8ff50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048e8ff50}, {0x40, 0xc00e567b70}, 0xc00e567ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048e8ff50}, {0x4e9d7a0, 0xc0933c3600}, 0xc047483c60, 0xc0c884dc80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048e8ff50}, 0xc074752ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc087816360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc087816360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805588 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016d6d2c0, {0x63c85b8, 0xc077013950}, 0xc074bb9290, 0x1, 0xc04e9010a8, 0x519bbc, 0xc05ef2a3c0, 0xc05c8dfc80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077013950}, 0x120, 0xc04e901230, 0x1, 0xc00d41b520, 0x18, 0xc05ef2a3c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016d6d2c0, {0x63c85b8, 0xc077013950}, 0xc05ef2a360, {{{{0x0, 0x0}, {0xc067a5c6c0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077013950}, 0x6352410, 0x17bb3a90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce3180, {0x63c85b8, 0xc077013950}, {0xc05ef2a240, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce3180, {0x63c85b8, 0xc077013950}, 0xc06034b900, 0xc04e9019f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077013950}, 0x660089, {0xc05ef2a240, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016efac00, {0x63c85b8, 0xc077013950}, 0xc0472aa000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0770138c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0770138c0}, {0xc04e9033f8, 0x660089}, 0xc04e903400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0770138c0}, {0x7fe42aae3ea0}, 0xc041ccde00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077013830}, 0xc041ccde00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}, 0xc06f236200, 0xc06f2648a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}, 0x3, 0xc077be3440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04e903980}}}, {0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}, 0xc06f236200, 0xc077be3440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc077013830}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077013830}, {0x40, 0xc04e903b70}, 0xc04e903ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}, 0x20301d, 0x20301d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc077013830}, {0x4e9d7a0, 0xc041ccde00}, 0xc06f236200, 0xc06f2648a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc077013830}, 0xc02b1d0e40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ef2a120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05ef2a120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802247 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc05655a030}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807014 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fefd400, {0x63c85b8, 0xc02e0c09f0}, 0xc02e5e0120, 0x1, 0xc0051bd0a8, 0x519bbc, 0xc0763ef800, 0xc0499115c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02e0c09f0}, 0x120, 0xc0051bd230, 0x763ef7a0, 0x0, 0x660069, 0xc0763ef800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fefd400, {0x63c85b8, 0xc02e0c09f0}, 0xc0763ef7a0, {{{{0x0, 0x0}, {0xc0145f7440, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02e0c09f0}, 0x6352410, 0x25ab66b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea3720, {0x63c85b8, 0xc02e0c09f0}, {0xc0763ef680, {0x1743011042c871a8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea3720, {0x63c85b8, 0xc02e0c09f0}, 0xc0465df680, 0xc0051bd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02e0c09f0}, 0x660089, {0xc0763ef680, {0x1743011042c871a8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff35c00, {0x63c85b8, 0xc02e0c09f0}, 0xc07de3c600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011042c871a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011042c871a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02e0c0870}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02e0c0870}, {0xc0051bf3f8, 0x660089}, 0xc0051bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02e0c0870}, {0x7230323231313244}, 0xc07de3c400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02e0c0780}, 0xc07de3c400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}, 0xc05665e120, 0xc02e5e14b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}, 0x3, 0xc0752f00c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0051bf980}}}, {0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}, 0xc05665e120, 0xc0752f00c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02e0c0780}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02e0c0780}, {0x40, 0xc0051bfb70}, 0xc0051bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02e0c0780}, {0x4e9d7a0, 0xc07de3c400}, 0xc05665e120, 0xc02e5e14b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02e0c0780}, 0xc01e2d82a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc034eb97a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc034eb97a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219743364 [runnable]: sync.runtime_SemacquireMutex(0x74b170, 0xa, 0xc076126eb8) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x3, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc00074b120, 0x2, 0x3df8d5a, 0xa563e9) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc033d71b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc05411ea80, 0xc033d71b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc033d71b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a523c0, 0xc0518f3280, 0xc0b2c03090, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc0b2c031a8, 0x4a, 0xc0b2c031a8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0b2c03040, 0xc058134f20, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b2c03040, {0xc0313b7880, 0x17, 0x79}, {0xc0331b20a0, 0x17, 0x4a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b2c01bc0, {0xc0331b20a0, 0x0, 0xc076127580}, 0x121ea30, 0x80) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b2c01bc0, {0xc0313b7880, 0x17, 0x4a}, {0xc0331b20a0, 0x16, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b2c01900, {0xc0331b20a0, 0x17, 0x4a}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0704d0108, {{0xc0760052f0, 0x16, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc003d14100, {{0xc0760052f0, 0x16, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc071b83800, {0x63c85b8, 0xc02557a060}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02557a060}, {0x64a6560, 0xc003d14100}, {0xc0760052f0, 0x3, 0x30}, {0x174301109e6bbca6, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02557a060}, {0x6466ed8, 0xc0704d0100}, {0xc0760052f0, 0x16, 0x30}, {0x174301109e6bbca6, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0a63e8df8}, {{0x174301109e6bbca6, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109e6bbca6, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05707f600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bc3921a6, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc017e9f000, {0x63c85b8, 0xc02557a060}, 0xc05707f600, 0xc07722e320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017e9f000, {0x63c85b8, 0xc02557a060}, 0xc05707f600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109e6bbca6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109e6bbca6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057891f20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057891f20}, {0xc07612d498, 0x660089}, 0xc07612d4a0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057891f20}, {0xc00102e4e0}, 0xc05707f500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057891ec0}, 0xc05707f500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc0490be4b0, {0x63c85b8, 0xc057891860}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc0490be4b0, {0x63c85b8, 0xc057891860}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc03ca984b0, {0x63c85b8, 0xc057891860}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc08478c000, {0x63c85b8, 0xc057891860}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc0c881e780) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc034107e40) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc0bb3b9200) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc08222e2c0, {0x63c8510, 0xc034107d40}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc086b49090, {0x63c8510, 0xc034107d40}, 0xc03ffa4630) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc034107a40}, 0xc08c4f4a80, 0xc03ffa4630, 0xc0d3ea4980, 0xc0d0960a80, 0xc04640dce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc034107a40}, 0xc04640d9f0, 0xc08c4f4a80, 0xc0e86f3140, {{0x63caae8, 0xc0266609a0}, 0x0}, 0xc0d0960a80) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc04640d300, {0x63c8510, 0xc034107a40}, 0xc04640d9f0, 0xc034107a40, {0x7fe42b790cf0, 0xc0e86f3140}, 0x0, 0xc03df189b8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc04640d300, {0x63c8510, 0xc034107a40}, 0xc04640d9f0, {0x7fe42b790cf0, 0xc0e86f3140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc04640d300, {0x63c85b8, 0xc04ae6bd70}, {{0x63f8d78, 0xc013f897c0}, {0xc0041649ad, 0x66}, 0x1, 0x1}, 0xc018d357a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc04640d300, {0x63c85b8, 0xc04ae6bd70}, {{0x63f8d78, 0xc013f897c0}, {0xc0041649ad, 0x66}, 0x1, 0x1}, 0xc018d357a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc04640d300, {0x63c85b8, 0xc04ae6bd70}, {0xc018d357a0, {0xc02013e920, 0x1, 0x1}, {0xc068a3f260, 0x10, 0x10}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc07d271c01, 0x0}, 0x0, {0xbf99571, 0xedb7a8351, 0x0}, 0x1}, 0xc04640d300, 0xc076133980, 0xc076133970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04640d300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04640d300, {0x63c8510, 0xc02964ee00}, 0xc071065dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc076a17180) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc02964ee00}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc076a17180) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219766201 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc056397290, 0xa5) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc06f79c000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc06f79c660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06cffac00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06cffac00, {0x63c8510, 0xc0563972c0}, 0xc06ff83dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0190b7370) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0563972c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0190b7370) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811085 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6f00, {0x63c85b8, 0xc046ac1d40}, 0xc041bf3548, 0x1, 0xc026dd50a8, 0x519bbc, 0xc0484c9020, 0xc058e3e180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046ac1d40}, 0x120, 0xc026dd5230, 0x484c8fc0, 0x0, 0x660069, 0xc0484c9020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6f00, {0x63c85b8, 0xc046ac1d40}, 0xc0484c8fc0, {{{{0x0, 0x0}, {0xc0518699e0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046ac1d40}, 0x6352410, 0x50b35b70) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c22d0, {0x63c85b8, 0xc046ac1d40}, {0xc0484c8ea0, {0x17430110725dba50, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c22d0, {0x63c85b8, 0xc046ac1d40}, 0xc083619d60, 0xc026dd59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046ac1d40}, 0x660089, {0xc0484c8ea0, {0x17430110725dba50, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4000, {0x63c85b8, 0xc046ac1d40}, 0xc08a3bb300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110725dba50, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110725dba50, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046ac1cb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046ac1cb0}, {0xc026dd73f8, 0x660089}, 0xc026dd7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046ac1cb0}, {0xc0015221e0}, 0xc08a3bb100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046ac1c20}, 0xc08a3bb100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}, 0xc056029e80, 0xc0185d7fb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}, 0x3, 0xc0131e0d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc026dd7980}}}, {0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}, 0xc056029e80, 0xc0131e0d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046ac1c20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046ac1c20}, {0x40, 0xc026dd7b70}, 0xc026dd7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046ac1c20}, {0x4e9d7a0, 0xc08a3bb100}, 0xc056029e80, 0xc0185d7fb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046ac1c20}, 0xc06ea442a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0484c8d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0484c8d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804668 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3df9b0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0619dec00, 0xc052867800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0619dec00, {0xc052867800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0619dec00, {0xc052867800, 0x735, 0xc06af5de00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0473ddd20, {0xc052867800, 0x7eda79, 0xc052867805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06d0e84e0, {0xc052867800, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2690f48, {0xc052867800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0558f2cf8, {0x631b900, 0xc0b2690f48}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0558f2a80, {0x7fe42ab59310, 0xc06d0e84e0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0558f2a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0558f2a80, {0xc020a54000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0299e2528, {0xc020a54000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc053f2c088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc053f2c088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc053f2c660, {0x634aac0, 0xc053f2c088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc053f2c000, 0xc0cd940190, 0xc070c0742b, 0xc070c0742d, {0x635bec0, 0xc00f462090}, 0xc000d6ae00, 0xc035fda0b8, 0xc070c07429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc053f2c000, {0x63c85b8, 0xc047e9a6f0}, 0xc0cd940180, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc048cbf038, 0x8}}, 0x0, 0xc047e9a690, 0xc047e9a6c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc047e9a6f0}, {0x643d810, 0xc06d0e84e0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04ecc5140}, {0x643d810, 0xc06d0e84e0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04ecc5140}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802120 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021d21000, {0x63c85b8, 0xc05454f260}, 0xc070038600, 0xc073bbb680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021d21000, {0x63c85b8, 0xc05454f260}, 0xc070038600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020b5935e, 0x0, 0x0}, 0xc024e28430, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020b5935e, 0x0, 0x0}, 0xc024e28430, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05454f050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05454f050}, {0xc075e53918, 0x660089}, 0xc075e53920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05454f050}, {0xc00102e4e0}, 0xc070038500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05454ef30}, 0xc070038500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc022b7ff80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc05454eae0}, {0xc00dfd36b0, 0x24}, 0x4a05b174c, 0xc01728cd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc08ccac680, 0xc00e4eac78}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc05454eae0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219707770 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc017f14640, {0x63c85b8, 0xc039b50c30}, 0xc020318798, 0x1, 0x70801810a4e29b, 0x37e11d600, 0xc06eb943c0, 0xc0ce793880) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc039b50c30}, 0x120, 0xc084ca12d0, 0x6eb94360, 0x0, 0x660069, 0xc06eb943c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc017f14640, {0x63c85b8, 0xc039b50c30}, 0xc06eb94360, {{{{0x0, 0x0}, {0xc0731326c0, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc039b50c30}, 0x6352230, 0xd1c8fbc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc017e6e550, {0x63c85b8, 0xc039b50c30}, {0xc06eb94240, {0x1743011074084f1a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc017e6e550, {0x63c85b8, 0xc039b50c30}, 0xc075272000, 0xc084ca1a90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc039b50c30}, 0x660089, {0xc06eb94240, {0x1743011074084f1a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017f21400, {0x63c85b8, 0xc039b50c30}, 0xc0535c1900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011074084f1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011074084f1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039b50ba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039b50ba0}, {0xc084ca3498, 0x660089}, 0xc084ca34a0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039b50ba0}, {0xc00102e4e0}, 0xc0535c1800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039b50b40}, 0xc0535c1800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc0606234a0, {0x63c85b8, 0xc039b50a80}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc0606234a0, {0x63c85b8, 0xc039b50a80}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc0d1c8fae0, {0x63c85b8, 0xc039b50a80}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc0a290ae00, {0x63c85b8, 0xc039b50a80}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColBatchScan).Next(0xc04630efa0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/colbatch_scan.go:107 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc0586d1180) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc071191500) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc04b691290, {0x63c8510, 0xc0586d1140}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0b2fa3ef0, {0x63c8510, 0xc0586d1140}, 0xc045f53ad0) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc0586d0f40}, 0xc02553c9a0, 0xc045f53ad0, 0xc06b559f00, 0xc03b633500, 0xc0799069e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc0586d0f40}, 0xc0799066f0, 0xc02553c9a0, 0xc05ff3a6c0, {{0x63caae8, 0xc08b440c40}, 0x0}, 0xc03b633500) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc079906000, {0x63c8510, 0xc0586d0f40}, 0xc0799066f0, 0xc0586d0f40, {0x7fe42b790cf0, 0xc05ff3a6c0}, 0x0, 0xc01f81e058) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc079906000, {0x63c8510, 0xc0586d0f40}, 0xc0799066f0, {0x7fe42b790cf0, 0xc05ff3a6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc079906000, {0x63c85b8, 0xc059891ce0}, {{0x63f8d78, 0xc04e299810}, {0xc06bc0800f, 0x66}, 0x1, 0x1}, 0xc02d238900, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc079906000, {0x63c85b8, 0xc059891ce0}, {{0x63f8d78, 0xc04e299810}, {0xc06bc0800f, 0x66}, 0x1, 0x1}, 0xc02d238900, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc079906000, {0x63c85b8, 0xc059891ce0}, {0xc02d238900, {0xc04d76aca0, 0x1, 0x1}, {0xc023347580, 0x10, 0x10}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc084968c05, 0x0}, 0x0, {0x1d30f689, 0xedb7a8350, 0x0}, 0x1}, 0xc079906000, 0xc084ca9980, 0xc084ca9970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc079906000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc079906000, {0x63c8510, 0xc00219b280}, 0xc058e31dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc034b44730) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc00219b280}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc034b44730) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219803644 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc0592edd40}, 0xc040e41e00, 0xc070ea5360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0592edd40}, 0xc040e41e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fae2e31, 0x0, 0x0}, 0xc0b4a481c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fae2e31, 0x0, 0x0}, 0xc0b4a481c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0592edcb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592edcb0}, {0xc04ed173f8, 0x660089}, 0xc04ed17400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0592edcb0}, {0x7fe13d162a38}, 0xc040e41c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0592edbc0}, 0xc040e41c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}, 0xc01ba25600, 0xc02d7ef9b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}, 0x3, 0xc039395a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04ed17980}}}, {0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}, 0xc01ba25600, 0xc039395a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0592edbc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0592edbc0}, {0x40, 0xc04ed17b70}, 0xc04ed17ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0592edbc0}, {0x4e9d7a0, 0xc040e41c00}, 0xc01ba25600, 0xc02d7ef9b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0592edbc0}, 0xc065e23980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0477745a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0477745a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805658 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc041ec5bf0}, 0xc04d91eb10, 0x1, 0xc001c990a8, 0x519bbc, 0xc093b98720, 0xc05866e960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041ec5bf0}, 0x120, 0xc001c99230, 0x93b986c0, 0x0, 0x660069, 0xc093b98720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc041ec5bf0}, 0xc093b986c0, {{{{0x0, 0x0}, {0xc057fd3200, 0x43}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041ec5bf0}, 0x6352410, 0x4207e260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc041ec5bf0}, {0xc093b985a0, {0x174301103f6cb70c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc041ec5bf0}, 0xc066ab8b40, 0xc001c999f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041ec5bf0}, 0x660089, {0xc093b985a0, {0x174301103f6cb70c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc041ec5bf0}, 0xc0607a5600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f6cb70c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f6cb70c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041ec5b30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ec5b30}, {0xc001c9b3f8, 0x660089}, 0xc001c9b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041ec5b30}, {0xc0015221e0}, 0xc0607a5000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041ec5aa0}, 0xc0607a5000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}, 0xc01e670ec0, 0xc03c3a37e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}, 0x3, 0xc03de36b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc001c9b980}}}, {0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}, 0xc01e670ec0, 0xc03de36b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041ec5aa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041ec5aa0}, {0x40, 0xc001c9bb70}, 0xc001c9bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041ec5aa0}, {0x4e9d7a0, 0xc0607a5000}, 0xc01e670ec0, 0xc03c3a37e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041ec5aa0}, 0xc046b4a0c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc093b98480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc093b98480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794947 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc024b3e6c0}, 0xc08e0a27e0, {0x646cc48, 0xc024b3e720}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc024b3e6c0}, 0xedb7a834e, 0xc061db4d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc024b3e6c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04a92fb00}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0185ccef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc04a92fb00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802034 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0310bf000, {0x63c85b8, 0xc0401b7ce0}, 0xc076191a00, 0xc050b1a1e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0310bf000, {0x63c85b8, 0xc0401b7ce0}, 0xc076191a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff4bfeabb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff4bfeabb, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0401b7c50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0401b7c50}, {0xc04dcf33f8, 0x660089}, 0xc04dcf3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0401b7c50}, {0x7fe3f3252190}, 0xc076191800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0401b7b90}, 0xc076191800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}, 0xc01f7e5640, 0xc01e5a4c60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}, 0x3, 0xc0ccfb9f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04dcf3980}}}, {0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}, 0xc01f7e5640, 0xc0ccfb9f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0401b7b90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0401b7b90}, {0x40, 0xc04dcf3b70}, 0xc04dcf3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0401b7b90}, {0x4e9d7a0, 0xc076191800}, 0xc01f7e5640, 0xc01e5a4c60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0401b7b90}, 0xc054521200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07b2a97a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07b2a97a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803643 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fe85c00, {0x63c85b8, 0xc0446fee40}, 0xc070957b00, 0xc062be5d60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fe85c00, {0x63c85b8, 0xc0446fee40}, 0xc070957b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102faead66, 0x0, 0x0}, 0xc0386b7500, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102faead66, 0x0, 0x0}, 0xc0386b7500, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0446fedb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0446fedb0}, {0xc0756eb3f8, 0x660089}, 0xc0756eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0446fedb0}, {0x0}, 0xc070957900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0446fed20}, 0xc070957900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}, 0xc0334e3960, 0xc026cf8a80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}, 0x3, 0xc03cd25640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0756eb980}}}, {0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}, 0xc0334e3960, 0xc03cd25640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0446fed20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0446fed20}, {0x40, 0xc0756ebb70}, 0xc0756ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0446fed20}, {0x4e9d7a0, 0xc070957900}, 0xc0334e3960, 0xc026cf8a80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0446fed20}, 0xc02fee2720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc047774480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc047774480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810385 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc079bb3d50, {0x63c85b8, 0xc040b0ac00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc040b0ac00}, {0x4f67ad2, 0x6446ff8}, 0xc088195fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219795510 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e068000, {0x63c85b8, 0xc026af7bc0}, 0xc09850b800, 0xc07a18f0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e068000, {0x63c85b8, 0xc026af7bc0}, 0xc09850b800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104a2798a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104a2798a9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc026af7ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc026af7ad0}, {0xc08ab0a6d8, 0x660089}, 0xc08ab0a6e0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc026af7ad0}, {0xc00102e4e0}, 0xc09850b700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc026af7a70}, 0xc09850b700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x7, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc00aee7b80, {0x63c85b8, 0xc026af7890}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc09f310000, {0x63c85b8, 0xc026af7890}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc09f310000, {0x63c8510, 0xc08ae71140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc09f310000, {0x63c8510, 0xc08ae71140}, 0xc0785ff280, {0x7fe42b790cf0, 0xc05f6f46c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc09f310000, {0x63c85b8, 0xc093d8ea20}, {{0x63f88f8, 0xc0785ff280}, {0xc08f83597d, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc09f310000, {0x63c85b8, 0xc093d8ea20}, {{0x63f88f8, 0xc0785ff280}, {0xc08f83597d, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0785ff280}, {0xc08f83597d, 0x23}, 0x0, 0x0}, {0x2f337be4, 0xedb7a834f, 0x0}, {0x2f33812f, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc09f310000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc09f310000, {0x63c8510, 0xc071b01c80}, 0xc0734fddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0875c1060) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc071b01c80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0875c1060) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219803217 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc072cadb90}, 0xc0b73d7200, 0xc0515b2960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc072cadb90}, 0xc0b73d7200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072cadb00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072cadb00}, {0xc0636d53f8, 0x660089}, 0xc0636d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072cadb00}, {0x7fe3c2f70200}, 0xc0b73d6e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072cada70}, 0xc0b73d6e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}, 0xc04618f360, 0xc027025770) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}, 0x3, 0xc0835f6bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0636d5980}}}, {0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}, 0xc04618f360, 0xc0835f6bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc072cada70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072cada70}, {0x40, 0xc0636d5b70}, 0xc0636d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc072cada70}, {0x4e9d7a0, 0xc0b73d6e00}, 0xc04618f360, 0xc027025770) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc072cada70}, 0xc0ae395ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0332dd200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0332dd200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810095 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0214da000, {0x63c85b8, 0xc07479dd70}, 0xc013df5f00, 0xc04b754500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc07479dd70}, 0xc013df5f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c2757b3, 0x0, 0x0}, 0xc039112980, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c2757b3, 0x0, 0x0}, 0xc039112980, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07479dce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07479dce0}, {0xc01aafb3f8, 0x660089}, 0xc01aafb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07479dce0}, {0xc0015221e0}, 0xc013df5d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07479dc50}, 0xc013df5d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}, 0xc05a888240, 0xc043d3efa8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}, 0x3, 0xc036bf2200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aafb980}}}, {0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}, 0xc05a888240, 0xc036bf2200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07479dc50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07479dc50}, {0x40, 0xc01aafbb70}, 0xc01aafbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07479dc50}, {0x4e9d7a0, 0xc013df5d00}, 0xc05a888240, 0xc043d3efa8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07479dc50}, 0xc025ec4ae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06cd2c000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06cd2c000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799999 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02ed8fd40}, 0xc07e11dcb0, {0x646cc48, 0xc02ed8fda0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02ed8fd40}, 0xedb7a834e, 0xc0552105a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02ed8fd40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06d59fe60}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc07e11def8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06d59fe60}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809645 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc000c98800, {0x63c85b8, 0xc044824810}, 0xc04e5b0500, 0xc024ef9d60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000c98800, {0x63c85b8, 0xc044824810}, 0xc04e5b0500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110669c2236, 0x0, 0x0}, 0xc062047a50, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110669c2236, 0x0, 0x0}, 0xc062047a50, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044824750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044824750}, {0xc01a0c53f8, 0x660089}, 0xc01a0c5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044824750}, {0x3ff0000000000000}, 0xc04e5b0200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0448246c0}, 0xc04e5b0200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}, 0xc0472a3d40, 0xc0337f9e60) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}, 0x3, 0xc0233e9600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a0c5980}}}, {0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}, 0xc0472a3d40, 0xc0233e9600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0448246c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0448246c0}, {0x40, 0xc01a0c5b70}, 0xc01a0c5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0448246c0}, {0x4e9d7a0, 0xc04e5b0200}, 0xc0472a3d40, 0xc0337f9e60) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0448246c0}, 0xc08d3b8b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc039bd6360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc039bd6360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810287 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01e7d3000, {0x63c85b8, 0xc076d947e0}, 0xc0a3d8f900, 0xc07a2f5040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e7d3000, {0x63c85b8, 0xc076d947e0}, 0xc0a3d8f900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d09600f, 0x0, 0x0}, 0xc02caf0a80, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d09600f, 0x0, 0x0}, 0xc02caf0a80, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076d94750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076d94750}, {0xc02807b3f8, 0x660089}, 0xc02807b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076d94750}, {0x7fe429f531a0}, 0xc0a3d8f700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076d946c0}, 0xc0a3d8f700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}, 0xc041caa5c0, 0xc0cb3874d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}, 0x3, 0xc04df1c0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02807b980}}}, {0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}, 0xc041caa5c0, 0xc04df1c0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076d946c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076d946c0}, {0x40, 0xc02807bb70}, 0xc02807bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076d946c0}, {0x4e9d7a0, 0xc0a3d8f700}, 0xc041caa5c0, 0xc0cb3874d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076d946c0}, 0xc04c333da0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060f5b7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060f5b7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811815 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ad17040, {0x63c85b8, 0xc03d563770}, 0xc02bde3cf8, 0x1, 0xc028a330a8, 0x519bbc, 0xc0146c7da0, 0xc071535b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc090f2a000, {0x63c85b8, 0xc03d563770}, 0xc0146c7e00, 0xc028a331a8, 0x3a, 0x50, 0x2, 0xc0146c7da0, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ad17040, {0x63c85b8, 0xc03d563770}, 0xc0146c7d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc073daf7a0, 0x2}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc03d563770}, 0x63523b0, 0x33a92048) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a557f90, {0x63c85b8, 0xc03d563770}, {0x0, {0x174301107b50848a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a557f90, {0x63c85b8, 0xc03d563770}, 0xc072301ae0, 0xc028a339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03d563770}, 0x660089, {0x0, {0x174301107b50848a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ad30800, {0x63c85b8, 0xc03d563770}, 0xc05513b800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107b50848a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107b50848a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301107b50848a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03d5636e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d5636e0}, {0xc028a353f8, 0x660089}, 0xc028a35400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03d5636e0}, {0x7fe3f7333998}, 0xc05513b600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03d5635f0}, 0xc05513b600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}, 0xc031e64b20, 0xc01dae56c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}, 0x3, 0xc065b05580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028a35980}}}, {0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}, 0xc031e64b20, 0xc065b05580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03d5635f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03d5635f0}, {0x40, 0xc028a35b70}, 0xc028a35ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03d5635f0}, {0x4e9d7a0, 0xc05513b600}, 0xc031e64b20, 0xc01dae56c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03d5635f0}, 0xc05b91dc80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0146c7c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0146c7c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812659 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219773761 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0202d8790, {0x63c85b8, 0xc07412b9e0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc03ea32c00, {0x63c85b8, 0xc07412b9e0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc03ea32c00, {0x63c8510, 0xc062edf100}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc03ea32c00, {0x63c8510, 0xc062edf100}, {0x63f7db8, 0x9bf6f48}, 0xc09622d088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc03ea32c00, {0x63c85b8, 0xc07ba46e70}, {{0x63f7db8, 0x9bf6f48}, {0xc0843d0c67, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc03ea32c00, {0x63c85b8, 0xc07ba46e70}, {{0x63f7db8, 0x9bf6f48}, {0xc0843d0c67, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0843d0c67, 0x6}, 0x0, 0x0}, {0x14dae344, 0xedb7a8350, 0x0}, {0x14daeb48, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03ea32c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03ea32c00, {0x63c8510, 0xc01789a680}, 0xc073097dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07b99fd80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc01789a680}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07b99fd80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812667 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812793 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805592 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc04818ac30}, 0xc032d85ce0, 0x1, 0xc07a60f0a8, 0x519bbc, 0xc05c5d5c80, 0xc05950ef00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04818ac30}, 0x120, 0xc07a60f230, 0x5c5d5c20, 0xc06c30fa00, 0x18, 0xc05c5d5c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc04818ac30}, 0xc05c5d5c20, {{{{0x0, 0x0}, {0xc058fcd680, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04818ac30}, 0x6352410, 0x6618a480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc04818ac30}, {0xc05c5d5b00, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc04818ac30}, 0xc081cd50e0, 0xc07a60f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04818ac30}, 0x660089, {0xc05c5d5b00, {0x174301103b676e2d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc04818ac30}, 0xc082858d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b676e2d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04818ab10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04818ab10}, {0xc07a6113f8, 0x660089}, 0xc07a611400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04818ab10}, {0x0}, 0xc082858b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04818aa80}, 0xc082858b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}, 0xc03a805c20, 0xc01607bc38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}, 0x3, 0xc04601ebc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07a611980}}}, {0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}, 0xc03a805c20, 0xc04601ebc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04818aa80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04818aa80}, {0x40, 0xc07a611b70}, 0xc07a611ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}, 0x203011, 0x203011) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04818aa80}, {0x4e9d7a0, 0xc082858b00}, 0xc03a805c20, 0xc01607bc38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04818aa80}, 0xc020124780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ef2b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05ef2b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800138 [runnable]: sync.runtime_SemacquireMutex(0x232aa0e, 0xc0, 0x7fe34e1f8300) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc06e265ba0, 0x300000002, 0xc06e265ba0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc00008d000, 0xc091738420, 0x4571d1, 0x424a070) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc06339e2c0, {0x0, 0x2128}, 0x0, 0xc07dfe9d78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc07dfe9b00, {0xffffffffffffff15, 0xc0917385b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc07dfe9b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc07dfe9b00, {0xc011a4ec00, 0x32, 0x72}, 0x12, 0x8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc07dfe9b00, {0xc07bd1dc20, 0x32, 0x72}, {0xc011a4ec00, 0x51c9860, 0xc011a4ec00}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc07dfe9b00, {0xc07bd1dc20, 0x32, 0x86}, {0xc011a4ec00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091eaa8e8, {0xc07bd1dc20, 0x32, 0x86}, {0xc011a4ec00, 0x32, 0x72}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091ea96c0, {0xc011a4ec00, 0x72, 0x86}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091ea96c0, {0xc07bd1dc20, 0x72, 0xc011a4ec00}, {0xc011a4ec00, 0xc09389b7a0, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091ea9400, {0xc011a4ec00, 0x32, 0x72}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0759a5328, {{0xc09389b7a0, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0509d5800, {{0xc0654a5590, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04f93b000, {0x63c85b8, 0xc02dd0ad80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02dd0ad80}, {0x64a6560, 0xc0509d5800}, {0xc0654a5590, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02dd0ad80}, {0x6466ed8, 0xc0759a5080}, {0xc0654a5590, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc066a04c18}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046785700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a33000, {0x63c85b8, 0xc02dd0ad80}, 0xc046785700, 0xc046ded9a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a33000, {0x63c85b8, 0xc02dd0ad80}, 0xc046785700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02dd0ac60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02dd0ac60}, {0xc0076c88f0, 0x660089}, 0xc0076c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02dd0ac60}, {0xc00102e4e0}, 0xc046785600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02dd0ac00}, 0xc046785600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805047 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b0ff900, {0x63c85b8, 0xc03f1c1f50}, 0xc03c2ef038, 0x1, 0xc06d3110a8, 0x519bbc, 0xc07013ba40, 0xc05eaf8840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03f1c1f50}, 0x120, 0xc06d311230, 0x7013b9e0, 0xc06e265d40, 0x18, 0xc07013ba40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b0ff900, {0x63c85b8, 0xc03f1c1f50}, 0xc07013b9e0, {{{{0x0, 0x0}, {0xc01b224000, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03f1c1f50}, 0x6352410, 0x18060228) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0fb090, {0x63c85b8, 0xc03f1c1f50}, {0xc07013b8c0, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0fb090, {0x63c85b8, 0xc03f1c1f50}, 0xc022382fa0, 0xc06d3119f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03f1c1f50}, 0x660089, {0xc07013b8c0, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b12c800, {0x63c85b8, 0xc03f1c1f50}, 0xc0622bae00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03f1c1ec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f1c1ec0}, {0xc06d3133f8, 0x660089}, 0xc06d313400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03f1c1ec0}, {0xc0015221e0}, 0xc0622bac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03f1c1e30}, 0xc0622bac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}, 0xc0664ffce0, 0xc03c2ef950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}, 0x3, 0xc060d7ff40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06d313980}}}, {0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}, 0xc0664ffce0, 0xc060d7ff40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03f1c1e30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f1c1e30}, {0x40, 0xc06d313b70}, 0xc06d313ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}, 0x203018, 0x203018) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03f1c1e30}, {0x4e9d7a0, 0xc0622bac00}, 0xc0664ffce0, 0xc03c2ef950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03f1c1e30}, 0xc0c3a60960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812671 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811150 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f22b400, {0x63c85b8, 0xc0473c4ff0}, 0xc06292f338, 0x1, 0x708018122a86c7, 0x37e11d600, 0xc0308d6720, 0xc0342b4de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0473c4ff0}, 0x120, 0xc00eb25230, 0x308d66c0, 0x0, 0x660069, 0xc0308d6720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f22b400, {0x63c85b8, 0xc0473c4ff0}, 0xc0308d66c0, {{{{0x0, 0x0}, {0xc060492a20, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0473c4ff0}, 0x6352410, 0x3f55a088) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f219b80, {0x63c85b8, 0xc0473c4ff0}, {0xc0308d65a0, {0x1743011074a011c7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f219b80, {0x63c85b8, 0xc0473c4ff0}, 0xc065f2f5e0, 0xc00eb259f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0473c4ff0}, 0x660089, {0xc0308d65a0, {0x1743011074a011c7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02072f400, {0x63c85b8, 0xc0473c4ff0}, 0xc045e94d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011074a011c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011074a011c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0473c4f00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473c4f00}, {0xc00eb273f8, 0x660089}, 0xc00eb27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0473c4f00}, {0x0}, 0xc045e94b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0473c4e70}, 0xc045e94b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}, 0xc01ed83ea0, 0xc06292f308) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}, 0x3, 0xc03763b380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00eb27980}}}, {0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}, 0xc01ed83ea0, 0xc03763b380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0473c4e70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473c4e70}, {0x40, 0xc00eb27b70}, 0xc00eb27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0473c4e70}, {0x4e9d7a0, 0xc045e94b00}, 0xc01ed83ea0, 0xc06292f308) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0473c4e70}, 0xc065e1ccc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0308d6480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0308d6480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812505 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x80, 0xc04142edc0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x0, 0xc044fe8a80, 0xc07da82e70) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04142edc0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc044fe8a80, {0xc06b7533b0, 0x1e, 0x6b}, {0xc068a3f020, 0x1e, 0x1e}, 0x0, 0x85) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0931583e8, {0xc06b7533b0, 0x1, 0x1e}, {0xc068a3f020, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0931583e8, {0xc06b7533b0, 0x1e, 0x6b}, {0xc068a3f020, 0x1e, 0x1e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0931571c0, {0xc068a3f020, 0x4a2372, 0x1e}, 0x464ed60, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0931571c0, {0xc06b7533b0, 0x1e, 0x1e}, {0xc068a3f020, 0x1d, 0x20}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093156f00, {0xc068a3f020, 0x1e, 0x1e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc087d32628, {{0xc068a3f000, 0x1d, 0x20}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc046607c00, {0x63c85b8, 0xc01c6160c0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01c6160c0}, {0x64a6618, 0xc087d32628}, {0xc068a3f000, 0x0, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01c6160c0}, {0x7fe429f83df0, 0xc034106b40}, {0xc068a3f000, 0x1d, 0x20}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto({0x63c85b8, 0xc01c6160c0}, {0x7fe429f83df0, 0xc034106b40}, {0xc068a3f000, 0xc07da83d10, 0x461247}, {0x0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:617 +0xc5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan.(*AbortSpan).Get(0x0, {0x63c85b8, 0xc01c6160c0}, {0x7fe429f83df0, 0xc034106b40}, {0x2c, 0xeb, 0x63, 0x36, 0xca, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan/abortspan.go:98 +0xdb github.com/cockroachdb/cockroach/pkg/kv/kvserver.checkIfTxnAborted({_, _}, {_, _}, {_, _}, {{{0x2c, 0xeb, 0x63, 0x36, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica.go:1958 +0xbc github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc07ae4fd00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:201 +0x37e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:644 +0x190 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, _, {{0x17430110a0c00774, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:612 +0x2da github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(_, {_, _}, {_, _}, _, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:428 +0x5f3 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc0255ac800, {0x63c85b8, 0xc01c6160c0}, {0xc06066c088, 0x8}, 0xc07ae4fd00, {{0x17430110a0c00774, 0x0, 0x0}, {0x1743011082f2a274, ...}}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:673 +0x1c5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0x0, {0x63c85b8, _}, {_, _}, _, {{{0x1742da4967dc0036, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_proposal.go:773 +0xaa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc0255ac800, {0x63c85b8, _}, _, _, {{{0x1742da4967dc0036, 0x0, 0x0}, 0x0, {0x1f, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:113 +0x1e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0255ac800, {0x63c85b8, 0xc01c6160c0}, 0xc07ae4fd00, 0xc05292f7c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:167 +0x7b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0255ac800, {0x63c85b8, 0xc01c6160c0}, 0xc07ae4fd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011082f2a274, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc018087d10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018087d10}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc018087d10}, {0xc00102e4e0}, 0xc07ae4fb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018087c20}, 0xc07ae4fb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0180870e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803458 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06c800, {0x63c85b8, 0xc077e67290}, 0xc0f1581b00, 0xc058cbe000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc077e67290}, 0xc0f1581b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ca14d44, 0x0, 0x0}, 0xc03b0d6e80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ca14d44, 0x0, 0x0}, 0xc03b0d6e80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077e67200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077e67200}, {0xc076589918, 0x660089}, 0xc076589920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077e67200}, {0xc00102e4e0}, 0xc0f1581a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077e671a0}, 0xc0f1581a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc03cb2f620}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc077e670e0}, {0xc00dfd36b0, 0x24}, 0x46c7bb22c, 0xc027908d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x65f3ef, 0xc0791a72c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc077e670e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803338 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fd49cc0, {0x63c85b8, 0xc01fb5d410}, 0xc0653d7290, 0x1, 0xc06fec90a8, 0x519bbc, 0xc044be5b60, 0xc074e76180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01fb5d410}, 0x120, 0xc06fec9230, 0x44be5b00, 0x0, 0x660069, 0xc044be5b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fd49cc0, {0x63c85b8, 0xc01fb5d410}, 0xc044be5b00, {{{{0x0, 0x0}, {0xc06ce6cfc0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01fb5d410}, 0x6352410, 0x41bc4db0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fc6bcc0, {0x63c85b8, 0xc01fb5d410}, {0xc044be59e0, {0x1743011025e86b69, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fc6bcc0, {0x63c85b8, 0xc01fb5d410}, 0xc077740c80, 0xc06fec99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01fb5d410}, 0x660089, {0xc044be59e0, {0x1743011025e86b69, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fda2000, {0x63c85b8, 0xc01fb5d410}, 0xc0c5e1d000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025e86b69, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025e86b69, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fb5d350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fb5d350}, {0xc06fecb3f8, 0x660089}, 0xc06fecb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fb5d350}, {0x7fe34bb6fee0}, 0xc0c5e1ce00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fb5d2c0}, 0xc0c5e1ce00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}, 0xc0276bcfa0, 0xc02684d428) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}, 0x3, 0xc065e5b940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06fecb980}}}, {0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}, 0xc0276bcfa0, 0xc065e5b940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01fb5d2c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fb5d2c0}, {0x40, 0xc06fecbb70}, 0xc06fecbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01fb5d2c0}, {0x4e9d7a0, 0xc0c5e1ce00}, 0xc0276bcfa0, 0xc02684d428) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01fb5d2c0}, 0xc029def4a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc044be47e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc044be47e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812717 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x80, 0xc01fce02c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc05f867180, 0xc03f7e9168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01fce02c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc05f867180, {0xc013a1b280, 0x28, 0x74}, {0xc0634cc150, 0x28, 0x6f}, 0x0, 0x88) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc090b186e0, {0xc013a1b280, 0x1, 0x6f}, {0xc0634cc150, 0xc090b17900, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc090b186e0, {0xc013a1b280, 0x28, 0x74}, {0xc0634cc150, 0x28, 0x6f}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc090b17bc0, {0xc0634cc150, 0x0, 0xc03f7e94e0}, 0x121ea30, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc090b17bc0, {0xc013a1b280, 0x28, 0x6f}, {0xc0634cc150, 0x27, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc090b17900, {0xc0634cc150, 0x28, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc084664b08, {{0xc076005800, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc08c835400, {{0xc076005800, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc058be0c00, {0x63c85b8, 0xc050520780}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc050520780}, {0x64a6560, 0xc08c835400}, {0xc076005800, 0xc00bc1fa40, 0x30}, {0x174301108f0bf0dc, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc050520780}, {0x6466ed8, 0xc084664b00}, {0xc076005800, 0x27, 0x30}, {0x174301108f0bf0dc, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0318b6408}, {{0x174301108f0bf0dc, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301108f0bf0dc, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0711f0100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110acd955dc, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02f354400, {0x63c85b8, 0xc050520780}, 0xc0711f0100, 0xc04fc11860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f354400, {0x63c85b8, 0xc050520780}, 0xc0711f0100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301108f0bf0dc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301108f0bf0dc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0505206f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0505206f0}, {0xc03f7ef3f8, 0x660089}, 0xc03f7ef400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0505206f0}, {0x1}, 0xc03486fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050520660}, 0xc03486fe00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}, 0xc00abd2fe0, 0xc017daec00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}, 0x3, 0xc06b3d6740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03f7ef980}}}, {0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}, 0xc00abd2fe0, 0xc06b3d6740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc050520660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050520660}, {0x40, 0xc03f7efb70}, 0xc03f7efab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc050520660}, {0x4e9d7a0, 0xc03486fe00}, 0xc00abd2fe0, 0xc017daec00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc050520660}, 0xc0289e7c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a4900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a4900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219776977 [runnable]: internal/poll.runtime_pollWait(0x7fde186b3210, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0835e0200, 0xc0336bc800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0835e0200, {0xc0336bc800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0835e0200, {0xc0336bc800, 0x735, 0xc05089aa00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0473dd850, {0xc0336bc800, 0x7eda79, 0xc0336bc805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc018df9980, {0xc0336bc800, 0x46a86f, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02f752db0, {0xc0336bc800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04db4d078, {0x631b900, 0xc02f752db0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04db4ce00, {0x7fe42ab59310, 0xc018df9980}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04db4ce00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04db4ce00, {0xc065b9b000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01ff38618, {0xc065b9b000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0610f6b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0610f6b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0610f70e0, {0x634aac0, 0xc0610f6b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0610f6a80, 0xc061c71220, 0xc07c0f142b, 0xc07c0f142d, {0x635bec0, 0xc06ef9ea20}, 0xc000d6ae00, 0xc041315e88, 0xc07c0f1429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0610f6a80, {0x63c85b8, 0xc06c1ce180}, 0xc061c71210, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc096c05038, 0x8}}, 0x0, 0xc06c1ce120, 0xc06c1ce150, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06c1ce180}, {0x643d810, 0xc018df9980}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04b94f470}, {0x643d810, 0xc018df9980}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04b94f470}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219798718 [runnable]: sync.runtime_SemacquireMutex(0xc092630690, 0x8f, 0x7fdc3dfc5380) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc092630538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc092630560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc056833680, {0xffffffffffffff15, 0xc0926306c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc056833680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc056833680, {0xc02c1a4c60, 0x32, 0x5d}, {0xc0745b3000, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc049bc9e98, {0xc02c1a4c60, 0x32, 0x5d}, {0xc0745b3000, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc049bc87c0, {0xc0745b3000, 0x79, 0x5d}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc049bc87c0, {0xc02c1a4c60, 0x79, 0xc0745b3000}, {0xc0745b3000, 0xc04299b1e0, 0x31}, 0xa3) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc049bc8500, {0xc0745b3000, 0x32, 0x79}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06c0363a8, {{0xc04299b1e0, 0x31, 0xa3}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0637bbc00, {{0xc0611eaf00, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc029e32000, {0x63c85b8, 0xc03c4c6f30}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03c4c6f30}, {0x64a6560, 0xc0637bbc00}, {0xc0611eaf00, 0x102e0000504c00, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03c4c6f30}, {0x6466ed8, 0xc06c036100}, {0xc0611eaf00, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023238918}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03eab7e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020578c00, {0x63c85b8, 0xc03c4c6f30}, 0xc03eab7e00, 0xc0189c5720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020578c00, {0x63c85b8, 0xc03c4c6f30}, 0xc03eab7e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03c4c6ea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03c4c6ea0}, {0xc0032fa8f0, 0x660089}, 0xc0032fa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03c4c6ea0}, {0xc00102e4e0}, 0xc03eab7d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03c4c6db0}, 0xc03eab7d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03c4c6c60}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811255 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc01b02eb70}, 0xc047c08048, 0x1, 0xc0289950a8, 0x519bbc, 0xc05ad0a9c0, 0xc04c492ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc01b02eb70}, 0x1e0, 0xc028995230, 0x5ad0a960, 0x0, 0x660069, 0xc05ad0a9c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc01b02eb70}, 0xc05ad0a960, {{{{0x0, 0x0}, {0xc039bbcfc0, 0x10a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc01b02eb70}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc01b02eb70}, {0xc03e79ea20, {0x1743011073c67f96, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc01b02eb70}, 0xc05ad73360, 0xc0289959f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01b02eb70}, 0x660089, {0xc03e79ea20, {0x1743011073c67f96, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01b02eb70}, 0xc01abe7d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011073c67f96, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011073c67f96, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b02e780}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b02e780}, {0xc0289973f8, 0x660089}, 0xc028997400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b02e780}, {0x7fe32e798418}, 0xc01abe7b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b02e660}, 0xc01abe7b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}, 0xc015834520, 0xc052b30258) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}, 0x3, 0xc019ff3200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028997980}}}, {0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}, 0xc015834520, 0xc019ff3200) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01b02e660}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b02e660}, {0x40, 0xc028997b70}, 0xc028997ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01b02e660}, {0x4e9d7a0, 0xc01abe7b00}, 0xc015834520, 0xc052b30258) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01b02e660}, 0xc020e0ab40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03e79e900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03e79e900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802944 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b26c000, {0x63c85b8, 0xc00cdb4420}, 0xc0b6df3900, 0xc01858bf40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc00cdb4420}, 0xc0b6df3900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011028eae57a, 0x0, 0x0}, 0xc0a204ce00, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011028eae57a, 0x0, 0x0}, 0xc0a204ce00, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc00cdb4300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00cdb4300}, {0xc0677cf3f8, 0x660089}, 0xc0677cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc00cdb4300}, {0x3c}, 0xc0b6df3600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc00cdb4240}, 0xc0b6df3600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}, 0xc029280520, 0xc03ed4b290) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}, 0x3, 0xc027ad91c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0677cf980}}}, {0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}, 0xc029280520, 0xc027ad91c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc00cdb4240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00cdb4240}, {0x40, 0xc0677cfb70}, 0xc0677cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc00cdb4240}, {0x4e9d7a0, 0xc0b6df3600}, 0xc029280520, 0xc03ed4b290) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc00cdb4240}, 0xc024169740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060c82360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060c82360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808340 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc01ff74de0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804964 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016c3b2c0, {0x63c85b8, 0xc04ef99b30}, 0xc0299e3d58, 0x1, 0x708017d83a302e, 0x37e11d600, 0xc06eaf2720, 0xc06ef1fa40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04ef99b30}, 0x120, 0xc071e79230, 0x2, 0xc0857fe820, 0x18, 0xc06eaf2720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016c3b2c0, {0x63c85b8, 0xc04ef99b30}, 0xc06eaf26c0, {{{{0x0, 0x0}, {0xc067a5d9e0, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ef99b30}, 0x6352410, 0x421b1860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce21e0, {0x63c85b8, 0xc04ef99b30}, {0xc06eaf25a0, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce21e0, {0x63c85b8, 0xc04ef99b30}, 0xc0325f95e0, 0xc071e799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ef99b30}, 0x660089, {0xc06eaf25a0, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016d2a000, {0x63c85b8, 0xc04ef99b30}, 0xc01aa11000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ef99aa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef99aa0}, {0xc071e7b3f8, 0x660089}, 0xc071e7b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ef99aa0}, {0x26}, 0xc01aa10e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ef99a10}, 0xc01aa10e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}, 0xc013beb900, 0xc0299e3d28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}, 0x3, 0xc06f13cfc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc071e7b980}}}, {0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}, 0xc013beb900, 0xc06f13cfc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ef99a10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef99a10}, {0x40, 0xc071e7bb70}, 0xc071e7bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ef99a10}, {0x4e9d7a0, 0xc01aa10e00}, 0xc013beb900, 0xc0299e3d28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ef99a10}, 0xc041535920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06f5abd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06f5abd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791305 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc043d66750}, 0xc05a792d00, 0xc02271e5a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc043d66750}, 0xc05a792d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ce103ca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ce103ca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043d66690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043d66690}, {0xc06d0427f8, 0x660089}, 0xc06d042800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043d66690}, {0xc00102e4e0}, 0xc05a792c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043d66630}, 0xc05a792c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc01e9ae6e0, {0x63c85b8, 0xc043d663f0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc08b700c00, {0x63c85b8, 0xc043d663f0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc08b700c00, {0x63c8510, 0xc040ff2d80}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc08b700c00, {0x63c8510, 0xc040ff2d80}, {0x63f7db8, 0x9bf6f48}, 0xc06d047088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc08b700c00, {0x63c85b8, 0xc044d7d140}, {{0x63f7db8, 0x9bf6f48}, {0xc026e935d6, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc08b700c00, {0x63c85b8, 0xc044d7d140}, {{0x63f7db8, 0x9bf6f48}, {0xc026e935d6, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc026e935d6, 0x6}, 0x0, 0x0}, {0x11e9304f, 0xedb7a834f, 0x0}, {0x11e93509, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08b700c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08b700c00, {0x63c8510, 0xc035daef40}, 0xc0768eddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03b6266d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc035daef40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03b6266d0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219800133 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc07c68c548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc07c68c5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07d2f80f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc081cbb180, {0xc08a9c79a0, 0x2b, 0x94}, {0xc06ae108a0, 0x2b, 0x55}, 0x1, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07d2f7f88, {0xc08a9c79a0, 0x1, 0x55}, {0xc06ae108a0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07d2f7f88, {0xc08a9c79a0, 0x2b, 0x94}, {0xc06ae108a0, 0x2b, 0x55}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07d2f76c0, {0xc06ae108a0, 0x55, 0x94}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07d2f76c0, {0xc08a9c79a0, 0x2b, 0xc06ae108a0}, {0xc06ae108a0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07d2f7400, {0xc06ae108a0, 0x2b, 0x55}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04272d708, {{0xc063b14330, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc072fc1000, {{0xc063b14330, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ecd9800, {0x63c85b8, 0xc05c416330}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc05c416330}, {0x64a6560, 0xc072fc1000}, {0xc063b14330, 0x4d545, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc05c416330}, {0x6466ed8, 0xc04272d700}, {0xc063b14330, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc030be1890}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04fef5300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a30000, {0x63c85b8, 0xc05c416330}, 0xc04fef5300, 0xc039b4a000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a30000, {0x63c85b8, 0xc05c416330}, 0xc04fef5300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05c4162a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05c4162a0}, {0xc0025128f0, 0x660089}, 0xc0025128f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05c4162a0}, {0xc00102e4e0}, 0xc04fef5200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05c416240}, 0xc04fef5200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d230}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812694 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812566 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802939 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc04be3e1e0}, 0xc01cb99b30, 0x1, 0xc067a116b8, 0x519bbc, 0xc0510e65a0, 0xc066798720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0c599ec90, {0x63c85b8, 0xc04be3e1e0}, 0xc06faad440, 0xc067a117b8, 0x510e65a0, 0x0, 0x0, 0xc0510e65a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc04be3e1e0}, 0xc0510e6540, {{{{0xc095ad3200, 0x25}, {0xc028415680, 0x3d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04be3e1e0}, 0x63525f0, 0x14c741a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc04be3e1e0}, {0x0, {0x1743011027ce83a9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc04be3e1e0}, 0xc0479a9e00, 0xc067a12000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04be3e1e0}, 0x660089, {0x0, {0x1743011027ce83a9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc04be3e1e0}, 0xc04c523f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011027ce83a9, 0x0, 0x0}, 0xc0448dc8c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011027ce83a9, 0x0, 0x0}, 0xc0448dc8c0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04be3e150}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04be3e150}, {0xc04c2efa08, 0x660089}, 0xc04c2efa10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04be3e150}, {0xc00102e4e0}, 0xc04c523e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04be3e0f0}, 0xc04c523e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc04be3e060}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04be3e060}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219484182 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bfa0878, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0570e2080, 0xc06e9c9000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0570e2080, {0xc06e9c9000, 0x2b98, 0x2b98}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0570e2080, {0xc06e9c9000, 0x2b98, 0xc038b1af00}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc071c3f3b8, {0xc06e9c9000, 0x7eda79, 0xc06e9c9005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01b1cdc80, {0xc06e9c9000, 0x2892, 0x2b98}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2690ab0, {0xc06e9c9000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc046a9daf8, {0x631b900, 0xc0b2690ab0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc046a9d880, {0x7fe42ab59310, 0xc01b1cdc80}, 0x2b98) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc046a9d880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc046a9d880, {0xc09034f000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03ce978f0, {0xc09034f000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05b74e088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05b74e088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05b74e660, {0x634aac0, 0xc05b74e088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05b74e000, 0xc0115fc6c0, 0xc059f6742b, 0xc059f6742d, {0x635bec0, 0xc033e76900}, 0xc000d6ae00, 0xc03216d268, 0xc059f67429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05b74e000, {0x63c85b8, 0xc04ec513e0}, 0xc0115fc6b0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0900f5037, 0x8}}, 0x0, 0xc04ec51350, 0xc04ec513b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04ec513e0}, {0x643d810, 0xc01b1cdc80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01d10cf60}, {0x643d810, 0xc01b1cdc80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01d10cf60}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811297 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016485540, {0x63c85b8, 0xc073763e00}, 0xc0a7bf6120, 0x1, 0xc0324470a8, 0x519bbc, 0xc06b277800, 0xc065619140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc073763e00}, 0x120, 0xc032447230, 0x6b2777a0, 0x0, 0x660069, 0xc06b277800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016485540, {0x63c85b8, 0xc073763e00}, 0xc06b2777a0, {{{{0x0, 0x0}, {0xc041c93c20, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073763e00}, 0x6352410, 0x55bf3258) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0164dc500, {0x63c85b8, 0xc073763e00}, {0xc06b277680, {0x17430110779db59f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0164dc500, {0x63c85b8, 0xc073763e00}, 0xc0ce7f3f40, 0xc0324479f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073763e00}, 0x660089, {0xc06b277680, {0x17430110779db59f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016515000, {0x63c85b8, 0xc073763e00}, 0xc08bd45200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110779db59f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110779db59f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073763d70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073763d70}, {0xc0324493f8, 0x660089}, 0xc032449400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073763d70}, {0xc0015221e0}, 0xc08bd45000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073763cb0}, 0xc08bd45000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}, 0xc05ba79300, 0xc0a7bf6840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}, 0x3, 0xc063c44540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032449980}}}, {0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}, 0xc05ba79300, 0xc063c44540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073763cb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073763cb0}, {0x40, 0xc032449b70}, 0xc032449ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073763cb0}, {0x4e9d7a0, 0xc08bd45000}, 0xc05ba79300, 0xc0a7bf6840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073763cb0}, 0xc03b0a8cc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06b277560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06b277560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803233 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc073606450}, 0xc065725c00, 0xc0cdfa6f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc073606450}, 0xc065725c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d169f57, 0x0, 0x0}, 0xc017e31120, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d169f57, 0x0, 0x0}, 0xc017e31120, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0736063c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0736063c0}, {0xc07085d3f8, 0x660089}, 0xc07085d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0736063c0}, {0xa56092ac091b0192}, 0xc065725a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073606330}, 0xc065725a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}, 0xc07a62a800, 0xc0453290e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}, 0x3, 0xc011a4adc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07085d980}}}, {0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}, 0xc07a62a800, 0xc011a4adc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073606330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073606330}, {0x40, 0xc07085db70}, 0xc07085dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073606330}, {0x4e9d7a0, 0xc065725a00}, 0xc07a62a800, 0xc0453290e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073606330}, 0xc078d92660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc038ab1560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc038ab1560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800271 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc04d9842a0}, 0xc016470cb0, {0x646cc48, 0xc04d984300}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc04d9842a0}, 0xedb7a834e, 0xc05129eb40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc04d9842a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0604ec6f0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc016470ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0604ec6f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809880 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc07971c270}, 0xc030a1a348, 0x1, 0xc01e9d50a8, 0x519bbc, 0xc0ac075260, 0xc083e2ade0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07971c270}, 0x120, 0xc01e9d5230, 0xac075200, 0x0, 0x660069, 0xc0ac075260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc07971c270}, 0xc0ac075200, {{{{0x0, 0x0}, {0xc09ecf0d80, 0x32}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07971c270}, 0x6352410, 0x603b9840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc07971c270}, {0xc0ac0750e0, {0x174301105ee9eb70, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc07971c270}, 0xc060d80c80, 0xc01e9d59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07971c270}, 0x660089, {0xc0ac0750e0, {0x174301105ee9eb70, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc07971c270}, 0xc083d9c100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105ee9eb70, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105ee9eb70, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07971c1e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07971c1e0}, {0xc01e9d73f8, 0x660089}, 0xc01e9d7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07971c1e0}, {0x7fe3bdd5bdc8}, 0xc05a3ad900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07971c150}, 0xc05a3ad900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}, 0xc03c8509e0, 0xc030a1b7e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}, 0x3, 0xc06f830900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01e9d7980}}}, {0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}, 0xc03c8509e0, 0xc06f830900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07971c150}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07971c150}, {0x40, 0xc01e9d7b70}, 0xc01e9d7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07971c150}, {0x4e9d7a0, 0xc05a3ad900}, 0xc03c8509e0, 0xc030a1b7e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07971c150}, 0xc0150dcde0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ac074ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0ac074ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812459 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021cd2000, {0x63c85b8, 0xc052b3bfb0}, 0xc01b2211e8, 0x1, 0xc0037770a8, 0x519bbc, 0xc0cc1605c0, 0xc05ae90420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04432cb40, {0x63c85b8, 0xc052b3bfb0}, 0xc04432cb78, 0xc0037771a8, 0x30, 0x40, 0xffffffffffffffff, 0xc0cc1605c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021cd2000, {0x63c85b8, 0xc052b3bfb0}, 0xc0cc160380, {{{{0xc05b281c20, 0x2}, {0xc051b779e0, 0x5}}, {{0xc03cc53560, 0xb}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc052b3bfb0}, 0x63521a0, 0x42012fc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c432c0, {0x63c85b8, 0xc052b3bfb0}, {0xc0a9515b00, {0x174301102079ef3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c432c0, {0x63c85b8, 0xc052b3bfb0}, 0xc075440be0, 0xc0037779f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc052b3bfb0}, 0x660089, {0xc0a9515b00, {0x174301102079ef3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd6000, {0x63c85b8, 0xc052b3bfb0}, 0xc07ae4e000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102079ef3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102079ef3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc052b3bf20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052b3bf20}, {0xc0037793f8, 0x660089}, 0xc003779400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc052b3bf20}, {0x1}, 0xc0435ddd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc052b3be00}, 0xc0435ddd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}, 0xc0adac6a60, 0xc04434fa40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}, 0x3, 0xc06051af00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003779980}}}, {0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}, 0xc0adac6a60, 0xc06051af00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc052b3be00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052b3be00}, {0x40, 0xc003779b70}, 0xc003779ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc052b3be00}, {0x4e9d7a0, 0xc0435ddd00}, 0xc0adac6a60, 0xc04434fa40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc052b3be00}, 0xc0369b3c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0a95159e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0a95159e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799942 [runnable]: sync.runtime_SemacquireMutex(0xc0954c2690, 0x8f, 0x7fe106325040) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0954c2538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0954c2560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc05423f680, {0xffffffffffffff15, 0xc0954c26c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc05423f680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc05423f680, {0xc06b5a22a0, 0x32, 0x6d}, {0xc01d212680, 0x32, 0x7f}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092738398, {0xc06b5a22a0, 0x32, 0x6d}, {0xc01d212680, 0x32, 0x7f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092736cc0, {0xc01d212680, 0x7f, 0x6d}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092736cc0, {0xc06b5a22a0, 0x7f, 0xc01d212680}, {0xc01d212680, 0xc0561a3e60, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092736a00, {0xc01d212680, 0x32, 0x7f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04f6ad9a8, {{0xc0561a3e60, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05fafc400, {{0xc06b6c3920, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0625b2000, {0x63c85b8, 0xc031a01380}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc031a01380}, {0x64a6560, 0xc05fafc400}, {0xc06b6c3920, 0x5d, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc031a01380}, {0x6466ed8, 0xc04f6ad700}, {0xc06b6c3920, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9a438}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc045019f00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209be800, {0x63c85b8, 0xc031a01380}, 0xc045019f00, 0xc0348de640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209be800, {0x63c85b8, 0xc031a01380}, 0xc045019f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031a00ff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031a00ff0}, {0xc0abc508f0, 0x660089}, 0xc0abc508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031a00ff0}, {0xc00102e4e0}, 0xc045019e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031a00f60}, 0xc045019e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01174b8f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802759 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bcd2000, {0x63c85b8, 0xc050361920}, 0xc045a31200, 0xc08ca88640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bcd2000, {0x63c85b8, 0xc050361920}, 0xc045a31200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025663314, 0x0, 0x0}, 0xc02f0ede20, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025663314, 0x0, 0x0}, 0xc02f0ede20, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050361890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050361890}, {0xc05ff453f8, 0x660089}, 0xc05ff45400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050361890}, {0x7fe13e89ce98}, 0xc045a31000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050361800}, 0xc045a31000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}, 0xc01d794440, 0xc016a05c68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}, 0x3, 0xc0519590c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05ff45980}}}, {0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}, 0xc01d794440, 0xc0519590c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc050361800}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050361800}, {0x40, 0xc05ff45b70}, 0xc05ff45ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc050361800}, {0x4e9d7a0, 0xc045a31000}, 0xc01d794440, 0xc016a05c68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc050361800}, 0xc0292dcde0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0479c9560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0479c9560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811072 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021614c80, {0x63c85b8, 0xc059891bc0}, 0xc0bb3a9350, 0x1, 0xc02859d0a8, 0x519bbc, 0xc01e767800, 0xc03fb3e960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc059891bc0}, 0x120, 0xc02859d230, 0x1e7677a0, 0x0, 0x660069, 0xc01e767800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021614c80, {0x63c85b8, 0xc059891bc0}, 0xc01e7677a0, {{{{0x0, 0x0}, {0xc00c90fe60, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc059891bc0}, 0x6352410, 0x3594b780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0215a9220, {0x63c85b8, 0xc059891bc0}, {0xc01e767680, {0x1743011072ddd270, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0215a9220, {0x63c85b8, 0xc059891bc0}, 0xc066747860, 0xc02859d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc059891bc0}, 0x660089, {0xc01e767680, {0x1743011072ddd270, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02164d800, {0x63c85b8, 0xc059891bc0}, 0xc0535c1400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011072ddd270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011072ddd270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc059891b00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059891b00}, {0xc02859f3f8, 0x660089}, 0xc02859f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc059891b00}, {0x33}, 0xc0535c1200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc059891a70}, 0xc0535c1200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}, 0xc043681b00, 0xc025b96000) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}, 0x3, 0xc0586d0ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02859f980}}}, {0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}, 0xc043681b00, 0xc0586d0ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc059891a70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc059891a70}, {0x40, 0xc02859fb06}, 0xc02859fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc059891a70}, {0x4e9d7a0, 0xc0535c1200}, 0xc043681b00, 0xc025b96000) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc059891a70}, 0xc0391f1ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc016157440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc016157440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805486 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff68780, {0x63c85b8, 0xc06cb941b0}, 0xc02844bed8, 0x1, 0x708017e49b56f0, 0x37e11d600, 0xc0938ac180, 0xc05fa6e060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06cb941b0}, 0x120, 0xc004b19230, 0x938ac120, 0x0, 0x660069, 0xc0938ac180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff68780, {0x63c85b8, 0xc06cb941b0}, 0xc0938ac120, {{{{0x0, 0x0}, {0xc04c926ea0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06cb941b0}, 0x6352410, 0x4040c4d0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0300060a0, {0x63c85b8, 0xc06cb941b0}, {0xc0938ac000, {0x17430110449da1e7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0300060a0, {0x63c85b8, 0xc06cb941b0}, 0xc0519219a0, 0xc004b199f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06cb941b0}, 0x660089, {0xc0938ac000, {0x17430110449da1e7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75000, {0x63c85b8, 0xc06cb941b0}, 0xc036124500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110449da1e7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110449da1e7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06cb94120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06cb94120}, {0xc004b1b3f8, 0x660089}, 0xc004b1b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06cb94120}, {0x1}, 0xc036124200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06cb94060}, 0xc036124200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}, 0xc0203d8040, 0xc02844be78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}, 0x3, 0xc0562dc700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004b1b980}}}, {0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}, 0xc0203d8040, 0xc0562dc700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06cb94060}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06cb94060}, {0x40, 0xc004b1bb70}, 0xc004b1bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06cb94060}, {0x4e9d7a0, 0xc036124200}, 0xc0203d8040, 0xc02844be78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06cb94060}, 0xc01219a240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093ad3b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc093ad3b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219767782 [IO wait]: internal/poll.runtime_pollWait(0x7fe42ab97f40, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc010a09580, 0xc005480000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc010a09580, {0xc005480000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc010a09580, {0xc005480000, 0x735, 0xc03438d540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03abf6940, {0xc005480000, 0x140, 0x13f}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc05c2fab40, {0xc005480000, 0x484427, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0c05c3de8, {0xc005480000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc082685e78, {0x631b900, 0xc0c05c3de8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc082685c00, {0x7fe42ab59310, 0xc05c2fab40}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc082685c00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc082685c00, {0xc08fb44000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc03e576480, {0xc08fb44000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05ebdab08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05ebdab08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05ebdb0e0, {0x634aac0, 0xc05ebdab08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05ebdaa80, 0xc07bf1fd40, 0xc032b4342b, 0xc032b4342d, {0x635bec0, 0xc0550f0c60}, 0xc000d6ae00, 0xc0530980d8, 0xc032b43429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05ebdaa80, {0x63c85b8, 0xc04b031ec0}, 0xc07bf1fd30, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0f0988037, 0x8}}, 0x0, 0xc04b031e60, 0xc04b031e90, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04b031ec0}, {0x643d810, 0xc05c2fab40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03c59c090}, {0x643d810, 0xc05c2fab40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03c59c090}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801068 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc024a03950}, {0xc0399e4000, 0x4, 0xc000691e58}, {0x0, {0x1743010fe626b659, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc024a03950}, 0xa9a, 0xc04876c120, 0x0, 0xc03f6414c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc024a03950}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802736 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016c3b2c0, {0x63c85b8, 0xc0590dcd80}, 0xc0181c3260, 0x1, 0xc0767610a8, 0x519bbc, 0xc03d7523c0, 0xc03e7fa3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0590dcd80}, 0x120, 0xc076761230, 0x3d752360, 0x0, 0x660069, 0xc03d7523c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016c3b2c0, {0x63c85b8, 0xc0590dcd80}, 0xc03d752360, {{{{0x0, 0x0}, {0xc04346ed80, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0590dcd80}, 0x6352410, 0x3214b630) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce21e0, {0x63c85b8, 0xc0590dcd80}, {0xc03d752240, {0x174301102df8528a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce21e0, {0x63c85b8, 0xc0590dcd80}, 0xc06f33a320, 0xc0767619f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0590dcd80}, 0x660089, {0xc03d752240, {0x174301102df8528a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016d2a000, {0x63c85b8, 0xc0590dcd80}, 0xc0449cbb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102df8528a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102df8528a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0590dccf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0590dccf0}, {0xc0767633f8, 0x660089}, 0xc076763400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0590dccf0}, {0x0}, 0xc0449cb900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0590dcc60}, 0xc0449cb900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}, 0xc022dbbd80, 0xc0667ba498) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}, 0x3, 0xc06efda280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076763980}}}, {0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}, 0xc022dbbd80, 0xc06efda280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0590dcc60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0590dcc60}, {0x40, 0xc076763b70}, 0xc076763ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0590dcc60}, {0x4e9d7a0, 0xc0449cb900}, 0xc022dbbd80, 0xc0667ba498) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0590dcc60}, 0xc01e4abaa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03f2f86c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03f2f86c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219781569 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0afa96a50, {0x63c85b8, 0xc01f165fb0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc07524d900, {0x63c85b8, 0xc01f165fb0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc07524d900, {0x63c8510, 0xc043957600}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc07524d900, {0x63c8510, 0xc043957600}, 0xc0456c7290, {0x7fe42b790cf0, 0xc0718d5140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc07524d900, {0x63c85b8, 0xc04317bd40}, {{0x63f88f8, 0xc0456c7290}, {0xc0385f38b5, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc07524d900, {0x63c85b8, 0xc04317bd40}, {{0x63f88f8, 0xc0456c7290}, {0xc0385f38b5, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0456c7290}, {0xc0385f38b5, 0x23}, 0x0, 0x0}, {0x12a93ab0, 0xedb7a834f, 0x0}, {0x12a9444c, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc07524d900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc07524d900, {0x63c8510, 0xc051e9ff00}, 0xc06fec7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08c6a5fd0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc051e9ff00}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08c6a5fd0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810999 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc06dca0e70}, 0xc08dac7590, 0x1, 0xc0275f30a8, 0x519bbc, 0xc04cae6fc0, 0xc083c61320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc08bab83c0, {0x63c85b8, 0xc06dca0e70}, 0xc07a1cd0e0, 0xc0275f31a8, 0x4cae6fc0, 0x0, 0x2, 0xc04cae6fc0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc06dca0e70}, 0xc04cae6f00, {{{{0xc093c1ed80, 0x39}, {0xc078937680, 0x5d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x30, {0x63c85b8, 0xc06dca0e70}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc06dca0e70}, {0xc085657e60, {0x17430110741db114, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc06dca0e70}, 0xc04544dc20, 0xc0275f39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06dca0e70}, 0x660089, {0xc085657e60, {0x17430110741db114, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc06dca0e70}, 0xc04e15de00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110741db114, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110741db114, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06dca0de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dca0de0}, {0xc0275f53f8, 0x660089}, 0xc0275f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06dca0de0}, {0x4148414d20455241}, 0xc04e15dc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06dca0d50}, 0xc04e15dc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}, 0xc047f1d1a0, 0xc056b266f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}, 0x3, 0xc06348eb40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0275f5980}}}, {0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}, 0xc047f1d1a0, 0xc06348eb40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06dca0d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06dca0d50}, {0x40, 0xc0275f5b70}, 0xc0275f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06dca0d50}, {0x4e9d7a0, 0xc04e15dc00}, 0xc047f1d1a0, 0xc056b266f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06dca0d50}, 0xc02879b080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc085657b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc085657b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808166 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc041e185d0}, 0xc049584500, 0xc028113720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc041e185d0}, 0xc049584500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aaf4c51, 0x0, 0x0}, 0xc019d6a740, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aaf4c51, 0x0, 0x0}, 0xc019d6a740, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041e183c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041e183c0}, {0xc00b77d3f8, 0x660089}, 0xc00b77d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041e183c0}, {0x7fe3f730dbc0}, 0xc049584300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041e18330}, 0xc049584300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}, 0xc06d02f1c0, 0xc022bd7170) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}, 0x3, 0xc034061d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b77d980}}}, {0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}, 0xc06d02f1c0, 0xc034061d80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041e18330}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041e18330}, {0x40, 0xc00b77db70}, 0xc00b77dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041e18330}, {0x4e9d7a0, 0xc049584300}, 0xc06d02f1c0, 0xc022bd7170) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041e18330}, 0xc08af2c900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07614bb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07614bb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800551 [semacquire]: sync.runtime_SemacquireMutex(0xc06cd001a0, 0x3, 0xc06cd001a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*Mutex).lockSlow(0xc00074a760) GOROOT/src/sync/mutex.go:138 +0x165 sync.(*Mutex).Lock(...) GOROOT/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc09606a338) GOROOT/src/sync/rwmutex.go:111 +0x36 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074a750, 0x11869e5, 0x7fdb6d5e1640, 0x7f66, 0x7fd52534c240) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc02551fa00, 0x7fdb6d5e1601, 0x7fdb6d5e1660, 0x4143, 0x4148) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc025573b80, {0x25e3c3b, 0x4143}, 0x0, 0xc065e1b8f8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc065e1b680, {0xffffffffffffff15, 0xc09606a5c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc065e1b680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc065e1b680, {0xc0476c7360, 0x2b, 0x4b}, 0x13, 0x44) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc065e1b680, {0xc023459d60, 0x2b, 0x4b}, {0xc0476c7360, 0x51c9860, 0xc0476c7360}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc065e1b680, {0xc023459d60, 0x2b, 0xa0}, {0xc0476c7360, 0x2b, 0x4b}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092a2ed98, {0xc023459d60, 0x2b, 0xa0}, {0xc0476c7360, 0x2b, 0x4b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092a2d6c0, {0xc0476c7360, 0x4b, 0xa0}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092a2d6c0, {0xc023459d60, 0x2b, 0xc0476c7360}, {0xc0476c7360, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092a2d400, {0xc0476c7360, 0x2b, 0x4b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0197d7708, {{0xc04464bf80, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc049e46500, {{0xc04464bf80, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc053cfa400, {0x63c85b8, 0xc06fc7bd40}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06fc7bd40}, {0x64a6560, 0xc049e46500}, {0xc04464bf80, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06fc7bd40}, {0x6466ed8, 0xc0197d7700}, {0xc04464bf80, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc068a03ae8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0747d7000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020af1c00, {0x63c85b8, 0xc06fc7bd40}, 0xc0747d7000, 0xc079c932c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020af1c00, {0x63c85b8, 0xc06fc7bd40}, 0xc0747d7000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06fc7bc50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fc7bc50}, {0xc0032fc8f0, 0x660089}, 0xc0032fc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06fc7bc50}, {0xc00102e4e0}, 0xc0747d6f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fc7bbf0}, 0xc0747d6f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7bb90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799769 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x6d) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xe0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc03d70acf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc066509500, {0xc02ba36980, 0x32, 0x77}, {0xc0307f2200, 0x32, 0x74}, 0x1, 0xac) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc03d70ab88, {0xc02ba36980, 0x1, 0x74}, {0xc0307f2200, 0x2c4, 0xc094ff4790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc03d70ab88, {0xc02ba36980, 0x32, 0x77}, {0xc0307f2200, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc03d70a2c0, {0xc0307f2200, 0x74, 0x77}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc03d70a2c0, {0xc02ba36980, 0x74, 0xc0307f2200}, {0xc0307f2200, 0xc04982ca80, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc03d70a000, {0xc0307f2200, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0759a4da8, {{0xc04982ca80, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0be3dde00, {{0xc064141da0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc081d3fc00, {0x63c85b8, 0xc029703230}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc029703230}, {0x64a6560, 0xc0be3dde00}, {0xc064141da0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc029703230}, {0x6466ed8, 0xc0759a4b00}, {0xc064141da0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04d848ea0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05075b800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209bc400, {0x63c85b8, 0xc029703230}, 0xc05075b800, 0xc074706000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209bc400, {0x63c85b8, 0xc029703230}, 0xc05075b800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0297031a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0297031a0}, {0xc09365a8f0, 0x660089}, 0xc09365a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0297031a0}, {0xc00102e4e0}, 0xc05075b700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029703140}, 0xc05075b700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01174b290}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811233 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc044fe7ce0}, 0xc024020c78, 0x1, 0x0, 0x1, 0xc00ec83020, 0xc04f573da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc044fe7ce0}, 0x120, 0xc00f306728, 0xec82fc0, 0x0, 0x660069, 0xc00ec83020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc044fe7ce0}, 0xc00ec82fc0, {{{{0x0, 0x0}, {0xc044b8d440, 0x2e}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc044fe7ce0}, 0x6352410, 0x6847e0f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc044fe7ce0}, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc044fe7ce0}, 0xc04bbddcc0, 0xc005be6ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc044fe7ce0}, 0x660089, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc044fe7ce0}, 0xc01abe6700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc044fe7aa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc044fe7aa0}, {0xc0562c48f0, 0x660089}, 0xc0562c48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc044fe7aa0}, {0xc00102e4e0}, 0xc01abe6600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc044fe7a40}, 0xc01abe6600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc044fe7590}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809857 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ad30800, {0x63c85b8, 0xc07718f2f0}, 0xc074123300, 0xc07bbee3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ad30800, {0x63c85b8, 0xc07718f2f0}, 0xc074123300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bfed68d, 0x0, 0x0}, 0xc03399e710, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bfed68d, 0x0, 0x0}, 0xc03399e710, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07718f260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07718f260}, {0xc00ecc13f8, 0x660089}, 0xc00ecc1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07718f260}, {0x7fe351f74048}, 0xc074123000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07718f0e0}, 0xc074123000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}, 0xc078f0eac0, 0xc072e1f7a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}, 0x3, 0xc0293cff80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ecc1980}}}, {0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}, 0xc078f0eac0, 0xc0293cff80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07718f0e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07718f0e0}, {0x40, 0xc00ecc1b70}, 0xc00ecc1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07718f0e0}, {0x4e9d7a0, 0xc074123000}, 0xc078f0eac0, 0xc072e1f7a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07718f0e0}, 0xc043a273e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc049d55d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc049d55d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807916 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc025026280, {0x63c85b8, 0xc0380d3950}, 0xc06f265548, 0x1, 0x0, 0x1, 0xc073d95b60, 0xc05b77f140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0380d3950}, 0x120, 0xc00620c728, 0x73d95b00, 0x0, 0x660069, 0xc073d95b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc025026280, {0x63c85b8, 0xc0380d3950}, 0xc073d95b00, {{{{0x0, 0x0}, {0xc0c2d550e0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0380d3950}, 0x6352410, 0x32430a48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0234e8a50, {0x63c85b8, 0xc0380d3950}, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0234e8a50, {0x63c85b8, 0xc0380d3950}, 0xc047209220, 0xc07c6c4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0380d3950}, 0x660089, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025034000, {0x63c85b8, 0xc0380d3950}, 0xc0ce8f3800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0380d38c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0380d38c0}, {0xc0032fe8f0, 0x660089}, 0xc0032fe8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0380d38c0}, {0xc00102e4e0}, 0xc0ce8f3700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0380d3860}, 0xc0ce8f3700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04e72e120}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812534 [runnable]: sync.runtime_SemacquireMutex(0x16cda16, 0x98, 0x11a5314) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074aa90, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xc025e9e020, 0xaacf, 0x16cda16, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01e9c1340, {0x13a580, 0x49fe}, 0x0, 0xc052307e78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc052307c00, {0xffffffffffffff15, 0xc00da9d0e8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc052307c00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc052307c00, {0xc04f2093b0, 0x4b, 0x4b}, 0xc052307c00, 0x48) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0x7fe42ab95398, {0xc021cb7880, 0xc052307c00, 0x0}, {0xc04f2093b0, 0xc085220ed8, 0x39}, 0x0, 0xf) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc085220d98, {0xc021cb7880, 0x1, 0x4b}, {0xc04f2093b0, 0xc08521f400, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc085220d98, {0xc021cb7880, 0x4b, 0x77}, {0xc04f2093b0, 0x4b, 0x4b}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc08521f6c0, {0xc04f2093b0, 0x0, 0xc00da9d4e0}, 0x121ea30, 0x20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc08521f6c0, {0xc021cb7880, 0x4b, 0x4b}, {0xc04f2093b0, 0x4a, 0x50}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc08521f400, {0xc04f2093b0, 0x4b, 0x4b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc075035b88, {{0xc02c1ba6e0, 0x4a, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc066dfc900, {{0xc02c1ba6e0, 0x4a, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc03fa58000, {0x63c85b8, 0xc01786da40}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01786da40}, {0x64a6560, 0xc066dfc900}, {0xc02c1ba6e0, 0xc00bc1fa40, 0x50}, {0x1743011086174da4, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01786da40}, {0x6466ed8, 0xc075035b80}, {0xc02c1ba6e0, 0x4a, 0x50}, {0x1743011086174da4, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc021e7c9f0}, {{0x1743011086174da4, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011086174da4, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06bbd5e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a3e4b2a4, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc000e4e800, {0x63c85b8, 0xc01786da40}, 0xc06bbd5e00, 0xc0742ead20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000e4e800, {0x63c85b8, 0xc01786da40}, 0xc06bbd5e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011086174da4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011086174da4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01786d890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01786d890}, {0xc00daa33f8, 0x660089}, 0xc00daa3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01786d890}, {0x7fe3316ccbe0}, 0xc06bbd5c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01786d710}, 0xc06bbd5c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}, 0xc0ccf7f480, 0xc038161218) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}, 0x3, 0xc02a52e280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00daa3980}}}, {0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}, 0xc0ccf7f480, 0xc02a52e280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01786d710}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01786d710}, {0x40, 0xc00daa3b70}, 0xc00daa3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01786d710}, {0x4e9d7a0, 0xc06bbd5c00}, 0xc0ccf7f480, 0xc038161218) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01786d710}, 0xc021af1e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdd200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc050cdd200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801936 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04ef9f6b0}, 0xc085655ef0, 0x1, 0x0, 0x1, 0xc050a6d6e0, 0xc01e6734a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04ef9f6b0}, 0x120, 0xc05a89c728, 0x50a6d680, 0x0, 0x660069, 0xc050a6d6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04ef9f6b0}, 0xc050a6d680, {{{{0x0, 0x0}, {0xc02957e480, 0xb6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ef9f6b0}, 0x6352410, 0x6541fca0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04ef9f6b0}, {0xc050a6cb40, {0x17430110206d867e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04ef9f6b0}, 0xc083fc1cc0, 0xc0766ceee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ef9f6b0}, 0x660089, {0xc050a6cb40, {0x17430110206d867e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04ef9f6b0}, 0xc045891d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110206d867e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110206d867e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ef9f620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef9f620}, {0xc0933e28f0, 0x660089}, 0xc0933e28f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ef9f620}, {0xc00102e4e0}, 0xc045891c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ef9f5c0}, 0xc045891c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04ef9ec00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219443228 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0455cca50, {0x63c85b8, 0xc0234c24b0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc06eb42000, {0x63c85b8, 0xc0234c24b0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc06eb42000, {0x63c8510, 0xc071366c40}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc06eb42000, {0x63c8510, 0xc071366c40}, {0x63f7db8, 0x9bf6f48}, 0xc0833f5088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc06eb42000, {0x63c85b8, 0xc043ba0cc0}, {{0x63f7db8, 0x9bf6f48}, {0xc094533ac1, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc06eb42000, {0x63c85b8, 0xc043ba0cc0}, {{0x63f7db8, 0x9bf6f48}, {0xc094533ac1, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc094533ac1, 0x6}, 0x0, 0x0}, {0xd385a82, 0xedb7a8350, 0x0}, {0xd386041, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06eb42000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06eb42000, {0x63c8510, 0xc065f4c200}, 0xc05f6cbdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc065748fd0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc065f4c200}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc065748fd0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219794814 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc049c22510, 0x5f) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc00ae1ea80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc00ae1f0e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc041a82c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc041a82c00, {0x63c8510, 0xc049c22540}, 0xc07658bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc088259820) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc049c22540}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc088259820) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219617814 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc03ef77810, {0x63c85b8, 0xc02478de30}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc052d55300, {0x63c85b8, 0xc02478de30}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc052d55300, {0x63c8510, 0xc0113fadc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc052d55300, {0x63c8510, 0xc0113fadc0}, {0x63f7db8, 0x9bf6f48}, 0xc07d1ed088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc052d55300, {0x63c85b8, 0xc018d6a000}, {{0x63f7db8, 0x9bf6f48}, {0xc09e5e4b1f, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc052d55300, {0x63c85b8, 0xc018d6a000}, {{0x63f7db8, 0x9bf6f48}, {0xc09e5e4b1f, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc09e5e4b1f, 0x6}, 0x0, 0x0}, {0x460d9bb, 0xedb7a8350, 0x0}, {0x460e19c, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc052d55300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc052d55300, {0x63c8510, 0xc044712d00}, 0xc08da99dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0121f3f00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc044712d00}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0121f3f00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810322 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01d898800, {0x63c85b8, 0xc01b6fbbf0}, 0xc056e61c00, 0xc06828d2c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc01b6fbbf0}, 0xc056e61c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c58104d, 0x0, 0x0}, 0xc03fc6eb50, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c58104d, 0x0, 0x0}, 0xc03fc6eb50, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b6fbb30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b6fbb30}, {0xc0246f33f8, 0x660089}, 0xc0246f3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b6fbb30}, {0x7fdd4a3590f8}, 0xc056e61a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b6fbaa0}, 0xc056e61a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}, 0xc088711740, 0xc047b80e10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}, 0x3, 0xc0118bd340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0246f3980}}}, {0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}, 0xc088711740, 0xc0118bd340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01b6fbaa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b6fbaa0}, {0x40, 0xc0246f3b70}, 0xc0246f3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01b6fbaa0}, {0x4e9d7a0, 0xc056e61a00}, 0xc088711740, 0xc047b80e10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01b6fbaa0}, 0xc017017200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0216a18c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0216a18c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807917 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc0380d3ad0}, 0xc0269f54b8, 0x1, 0x0, 0x1, 0xc073d95da0, 0xc05c8003c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0380d3ad0}, 0x120, 0xc00e7e2728, 0x73d95d40, 0x0, 0x660069, 0xc073d95da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc0380d3ad0}, 0xc073d95d40, {{{{0x0, 0x0}, {0xc07afbf200, 0x1c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0380d3ad0}, 0x6352410, 0x32430a50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc0380d3ad0}, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc0380d3ad0}, 0xc02e01a320, 0xc07c6c4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0380d3ad0}, 0x660089, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0380d3ad0}, 0xc0ce8f3a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0380d3a10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0380d3a10}, {0xc0032fe8f0, 0x660089}, 0xc0032fe8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0380d3a10}, {0xc00102e4e0}, 0xc0ce8f3900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0380d39b0}, 0xc0ce8f3900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04e72e180}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811334 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021def400, {0x63c85b8, 0xc03ed18540}, 0xc056b27bd8, 0x1, 0x70801811f9cda7, 0x37e11d600, 0xc03acb9c80, 0xc09d4fb5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03ed18540}, 0x120, 0xc02944b230, 0x3acb9c20, 0x0, 0x660069, 0xc03acb9c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021def400, {0x63c85b8, 0xc03ed18540}, 0xc03acb9c20, {{{{0x0, 0x0}, {0xc0639cfb00, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03ed18540}, 0x6352410, 0x1b309e48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3dc20, {0x63c85b8, 0xc03ed18540}, {0xc03acb9b00, {0x174301107494b7d8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3dc20, {0x63c85b8, 0xc03ed18540}, 0xc0602f5d60, 0xc02944b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03ed18540}, 0x660089, {0xc03acb9b00, {0x174301107494b7d8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021ebd800, {0x63c85b8, 0xc03ed18540}, 0xc0659ccc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107494b7d8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107494b7d8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03ed184b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ed184b0}, {0xc02944d3f8, 0x660089}, 0xc02944d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03ed184b0}, {0xc0015221e0}, 0xc0659cca00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03ed18420}, 0xc0659cca00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}, 0xc047f1dc80, 0xc056b27ba8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}, 0x3, 0xc06348fe80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02944d980}}}, {0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}, 0xc047f1dc80, 0xc06348fe80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03ed18420}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03ed18420}, {0x40, 0xc02944db70}, 0xc02944dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03ed18420}, {0x4e9d7a0, 0xc0659cca00}, 0xc047f1dc80, 0xc056b27ba8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03ed18420}, 0xc028e9d740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03acb99e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03acb99e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811382 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016c3b2c0, {0x63c85b8, 0xc092ec4570}, 0xc049b45098, 0x1, 0xc00ef930a8, 0x519bbc, 0xc06dcb6960, 0xc033c0bb60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc092ec4570}, 0x120, 0xc00ef93230, 0x6dcb6900, 0x0, 0x660069, 0xc06dcb6960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016c3b2c0, {0x63c85b8, 0xc092ec4570}, 0xc06dcb6900, {{{{0x0, 0x0}, {0xc0d08afb00, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc092ec4570}, 0x6352410, 0x202c4d58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce21e0, {0x63c85b8, 0xc092ec4570}, {0xc06dcb67e0, {0x1743011076474074, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce21e0, {0x63c85b8, 0xc092ec4570}, 0xc070ea4960, 0xc00ef939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc092ec4570}, 0x660089, {0xc06dcb67e0, {0x1743011076474074, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016d2a000, {0x63c85b8, 0xc092ec4570}, 0xc01ac98b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc092ec44e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092ec44e0}, {0xc00ef953f8, 0x660089}, 0xc00ef95400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc092ec44e0}, {0x7fe426b0a1a8}, 0xc04dc35e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc092ec4450}, 0xc04dc35e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}, 0xc05fc32620, 0xc06613d350) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}, 0x3, 0xc06c926e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ef95980}}}, {0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}, 0xc05fc32620, 0xc06c926e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc092ec4450}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092ec4450}, {0x40, 0xc00ef95b70}, 0xc00ef95ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc092ec4450}, {0x4e9d7a0, 0xc04dc35e00}, 0xc05fc32620, 0xc06613d350) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc092ec4450}, 0xc03d498780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06dcb66c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06dcb66c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806770 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021cd37c0, {0x63c85b8, 0xc08dd7c420}, 0xc08fc7a330, 0x1, 0xc002bd30a8, 0x519bbc, 0xc0d22c5da0, 0xc086d97260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08dd7c420}, 0x120, 0xc002bd3230, 0xd22c5d40, 0x0, 0x660069, 0xc0d22c5da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021cd37c0, {0x63c85b8, 0xc08dd7c420}, 0xc0d22c5d40, {{{{0x0, 0x0}, {0xc0627030e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08dd7c420}, 0x6352410, 0x304bbbd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c43ea0, {0x63c85b8, 0xc08dd7c420}, {0xc0d22c5c20, {0x174301104ec6aed2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c43ea0, {0x63c85b8, 0xc08dd7c420}, 0xc04baade00, 0xc002bd39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08dd7c420}, 0x660089, {0xc0d22c5c20, {0x174301104ec6aed2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021d20400, {0x63c85b8, 0xc08dd7c420}, 0xc08b379100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104ec6aed2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104ec6aed2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08dd7c390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08dd7c390}, {0xc002bd53f8, 0x660089}, 0xc002bd5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08dd7c390}, {0x110}, 0xc08b378f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08dd7c300}, 0xc08b378f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}, 0xc08e0fa220, 0xc01636baa0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}, 0x3, 0xc08c99f340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002bd5980}}}, {0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}, 0xc08e0fa220, 0xc08c99f340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08dd7c300}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08dd7c300}, {0x40, 0xc002bd5b70}, 0xc002bd5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08dd7c300}, {0x4e9d7a0, 0xc08b378f00}, 0xc08e0fa220, 0xc01636baa0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08dd7c300}, 0xc0305c4f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0457f07e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0457f07e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811381 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016f5da40, {0x63c85b8, 0xc092ec4720}, 0xc06613d3e0, 0x1, 0x708018132fa41a, 0x37e11d600, 0xc06dcb6ba0, 0xc0604b6ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc092ec4720}, 0x120, 0xc0315ef230, 0x6dcb6b40, 0x0, 0x660069, 0xc06dcb6ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016f5da40, {0x63c85b8, 0xc092ec4720}, 0xc06dcb6b40, {{{{0x0, 0x0}, {0xc073d95560, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc092ec4720}, 0x6352410, 0x202c4d60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016fda500, {0x63c85b8, 0xc092ec4720}, {0xc06dcb6a20, {0x1743011076474074, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016fda500, {0x63c85b8, 0xc092ec4720}, 0xc049d0cf00, 0xc0315ef9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc092ec4720}, 0x660089, {0xc06dcb6a20, {0x1743011076474074, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017050000, {0x63c85b8, 0xc092ec4720}, 0xc01ac99600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011076474074, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc092ec4690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092ec4690}, {0xc0315f13f8, 0x660089}, 0xc0315f1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc092ec4690}, {0x7fe3c2f26d98}, 0xc01ac99400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc092ec4600}, 0xc01ac99400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}, 0xc05fc32680, 0xc06613d3b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}, 0x3, 0xc06c926f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0315f1980}}}, {0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}, 0xc05fc32680, 0xc06c926f40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc092ec4600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc092ec4600}, {0x40, 0xc0315f1b70}, 0xc0315f1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc092ec4600}, {0x4e9d7a0, 0xc01ac99400}, 0xc05fc32680, 0xc06613d3b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc092ec4600}, 0xc03d498960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06dcb65a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06dcb65a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799217 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc093cb0548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc093cb05b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0399225f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0539a3180, {0xc05ff93780, 0x2b, 0x74}, {0xc03a72b500, 0x2b, 0x6c}, 0x1, 0x25) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc039922488, {0xc05ff93780, 0x1, 0x6c}, {0xc03a72b500, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc039922488, {0xc05ff93780, 0x2b, 0x74}, {0xc03a72b500, 0x2b, 0x6c}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc039921bc0, {0xc03a72b500, 0x6c, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc039921bc0, {0xc05ff93780, 0x2b, 0xc03a72b500}, {0xc03a72b500, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc039921900, {0xc03a72b500, 0x2b, 0x6c}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06c037188, {{0xc0654a4e70, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc01247f400, {{0xc0654a4e70, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc041f02800, {0x63c85b8, 0xc0a5b24810}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0a5b24810}, {0x64a6560, 0xc01247f400}, {0xc0654a4e70, 0x20907, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0a5b24810}, {0x6466ed8, 0xc06c037180}, {0xc0654a4e70, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc035a18c48}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc070752b00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0207d8800, {0x63c85b8, 0xc0a5b24810}, 0xc070752b00, 0xc01d3e3cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0207d8800, {0x63c85b8, 0xc0a5b24810}, 0xc070752b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0a5b24780}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0a5b24780}, {0xc0d68328f0, 0x660089}, 0xc0d68328f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0a5b24780}, {0xc00102e4e0}, 0xc070752a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0a5b24720}, 0xc070752a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc78ed0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812011 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc04c4ccb10}, 0xc03c5398d8, 0x1, 0x7080181b76179c, 0x37e11d600, 0xc027eb6420, 0xc05aa3be60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04c4ccb10}, 0x1e0, 0xc0328ca178, 0x27eb63c0, 0x0, 0x660069, 0xc027eb6420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc04c4ccb10}, 0xc027eb63c0, {{{{0x0, 0x0}, {0xc0af5be480, 0x70}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04c4ccb10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc04c4ccb10}, {0xc0130758c0, {0x174301107c01b237, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc04c4ccb10}, 0xc053b560a0, 0xc07cdf4938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c4ccb10}, 0x660089, {0xc0130758c0, {0x174301107c01b237, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc04c4ccb10}, 0xc04b09b000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107c01b237, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107c01b237, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c4cca80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c4cca80}, {0xc07cdf6340, 0x660089}, 0xc07cdf6348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c4cca80}, {0xc00102e4e0}, 0xc04b09af00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c4cca20}, 0xc04b09af00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc04c4cc5a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801853 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc029f34420}, 0xc0657b2600, 0xc07229e8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc029f34420}, 0xc0657b2600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101df83bba, 0x0, 0x0}, 0xc035a40290, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101df83bba, 0x0, 0x0}, 0xc035a40290, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029f34390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029f34390}, {0xc05725f3f8, 0x660089}, 0xc05725f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029f34390}, {0x40e41bdfa509edee}, 0xc0657b2400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029f342a0}, 0xc0657b2400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}, 0xc0263b6000, 0xc09e286348) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}, 0x3, 0xc0c7fd2f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05725f980}}}, {0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}, 0xc0263b6000, 0xc0c7fd2f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029f342a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029f342a0}, {0x40, 0xc05725fb08}, 0xc05725fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029f342a0}, {0x4e9d7a0, 0xc0657b2400}, 0xc0263b6000, 0xc09e286348) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029f342a0}, 0xc04df26480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03afded80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03afded80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803813 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc029ccdaa0}, 0xc04dd25470, 0x1, 0xc075a250a8, 0x519bbc, 0xc0497ada40, 0xc049fe6cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc029ccdaa0}, 0x120, 0xc075a25230, 0x497ad9e0, 0x0, 0x660069, 0xc0497ada40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc029ccdaa0}, 0xc0497ad9e0, {{{{0x0, 0x0}, {0xc06d4c8000, 0x1c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc029ccdaa0}, 0x6352410, 0x1f8e0378) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc029ccdaa0}, {0xc0497ad8c0, {0x174301102fcc5c7a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc029ccdaa0}, 0xc06cf4e1e0, 0xc075a259f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc029ccdaa0}, 0x660089, {0xc0497ad8c0, {0x174301102fcc5c7a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc029ccdaa0}, 0xc002b0fe00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fcc5c7a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fcc5c7a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc029ccda10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029ccda10}, {0xc075a273f8, 0x660089}, 0xc075a27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc029ccda10}, {0x0}, 0xc002b0fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc029ccd920}, 0xc002b0fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}, 0xc07753a460, 0xc05fcec930) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}, 0x3, 0xc064cb3a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075a27980}}}, {0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}, 0xc07753a460, 0xc064cb3a40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc029ccd920}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc029ccd920}, {0x40, 0xc075a27b70}, 0xc075a27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc029ccd920}, {0x4e9d7a0, 0xc002b0fc00}, 0xc07753a460, 0xc05fcec930) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc029ccd920}, 0xc064944060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04adbf200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04adbf200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807728 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ff35c00, {0x63c85b8, 0xc0380d3f80}, 0xc09a562000, 0xc022382b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff35c00, {0x63c85b8, 0xc0380d3f80}, 0xc09a562000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011059e7b505, 0x0, 0x0}, 0xc010406d30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011059e7b505, 0x0, 0x0}, 0xc010406d30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0380d3ec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0380d3ec0}, {0xc00ecc13f8, 0x660089}, 0xc00ecc1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0380d3ec0}, {0x7fe39458cd40}, 0xc0ce8f3e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0380d3d70}, 0xc0ce8f3e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}, 0xc081bc8ce0, 0xc09c3227c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}, 0x3, 0xc04a090e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ecc1980}}}, {0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}, 0xc081bc8ce0, 0xc04a090e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0380d3d70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0380d3d70}, {0x40, 0xc00ecc1b70}, 0xc00ecc1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0380d3d70}, {0x4e9d7a0, 0xc0ce8f3e00}, 0xc081bc8ce0, 0xc09c3227c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0380d3d70}, 0xc015be0300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0718357a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0718357a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809164 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc04ced9500}, 0xc043b2b000, 0xc0c59ae460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04ced9500}, 0xc043b2b000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063920fbd, 0x0, 0x0}, 0xc03974bc30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063920fbd, 0x0, 0x0}, 0xc03974bc30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ced9470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ced9470}, {0xc0d1bd5a08, 0x660089}, 0xc0d1bd5a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ced9470}, {0xc00102e4e0}, 0xc043b2af00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ced9410}, 0xc043b2af00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc04ced9380}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04ced9380}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219770373 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc01e9ae630, {0x63c85b8, 0xc06fcdb4a0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc08f538000, {0x63c85b8, 0xc06fcdb4a0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc08f538000, {0x63c8510, 0xc06ac7ff00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc08f538000, {0x63c8510, 0xc06ac7ff00}, 0xc0332c8cb0, {0x7fe42b790cf0, 0xc0bc3d5bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc08f538000, {0x63c85b8, 0xc044d7ca20}, {{0x63f88f8, 0xc0332c8cb0}, {0xc09bfa6fcc, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc08f538000, {0x63c85b8, 0xc044d7ca20}, {{0x63f88f8, 0xc0332c8cb0}, {0xc09bfa6fcc, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc0332c8cb0}, {0xc09bfa6fcc, 0x23}, 0x0, 0x0}, {0x23775451, 0xedb7a834f, 0x0}, {0x23775db2, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc08f538000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc08f538000, {0x63c8510, 0xc058763180}, 0xc0bbce9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06b537c20) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc058763180}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc06b537c20) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808777 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6140, {0x63c85b8, 0xc0307c0b40}, 0xc027813008, 0x1, 0xc006d250a8, 0x519bbc, 0xc0629da960, 0xc05c3b2840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0307c0b40}, 0x120, 0xc006d25230, 0x629da900, 0x0, 0x660069, 0xc0629da960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6140, {0x63c85b8, 0xc0307c0b40}, 0xc0629da900, {{{{0x0, 0x0}, {0xc090f2b8c0, 0xa}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0307c0b40}, 0x63526b0, 0x1fa2e290) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddbd0, {0x63c85b8, 0xc0307c0b40}, {0xc0629da7e0, {0x1743011060435abe, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddbd0, {0x63c85b8, 0xc0307c0b40}, 0xc044f78280, 0xc006d259f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0307c0b40}, 0x660089, {0xc0629da7e0, {0x1743011060435abe, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099c00, {0x63c85b8, 0xc0307c0b40}, 0xc025102b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011060435abe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011060435abe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0307c0ab0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0307c0ab0}, {0xc006d273f8, 0x660089}, 0xc006d27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0307c0ab0}, {0xc0a1ffa1b3cbee08}, 0xc025102900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0307c09f0}, 0xc025102900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}, 0xc0ce0eb740, 0xc02f7dd440) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}, 0x3, 0xc0c1ccc6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006d27980}}}, {0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}, 0xc0ce0eb740, 0xc0c1ccc6c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0307c09f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0307c09f0}, {0x40, 0xc006d27b70}, 0xc006d27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0307c09f0}, {0x4e9d7a0, 0xc025102900}, 0xc0ce0eb740, 0xc02f7dd440) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0307c09f0}, 0xc03b1da900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0629da5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0629da5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807369 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc0260b8d80}, 0xc0984be200, 0xc095d5dae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0260b8d80}, 0xc0984be200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105731816a, 0x0, 0x0}, 0xc090690730, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105731816a, 0x0, 0x0}, 0xc090690730, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0260b8cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0260b8cf0}, {0xc00374d3f8, 0x660089}, 0xc00374d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0260b8cf0}, {0x7fe3e2171430}, 0xc0984be000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0260b8c60}, 0xc0984be000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}, 0xc07b4d70a0, 0xc06605a600) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}, 0x3, 0xc08872b540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00374d980}}}, {0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}, 0xc07b4d70a0, 0xc08872b540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0260b8c60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0260b8c60}, {0x40, 0xc00374db70}, 0xc00374dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0260b8c60}, {0x4e9d7a0, 0xc0984be000}, 0xc07b4d70a0, 0xc06605a600) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0260b8c60}, 0xc0534df4a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07b041b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc07b041b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803459 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc05ca98dc0, {0x63c85b8, 0xc020a924e0}, 0xc07bc934d0, 0x1, 0xc0677cd0a8, 0x519bbc, 0xc038a94a80, 0xc0378abda0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc020a924e0}, 0x120, 0xc0677cd230, 0x38a94a20, 0x0, 0x660069, 0xc038a94a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc05ca98dc0, {0x63c85b8, 0xc020a924e0}, 0xc038a94a20, {{{{0x0, 0x0}, {0xc07bf285a0, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc020a924e0}, 0x6352410, 0x210ac248) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc045008410, {0x63c85b8, 0xc020a924e0}, {0xc038a94900, {0x174301102bb82fa6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc045008410, {0x63c85b8, 0xc020a924e0}, 0xc057f901e0, 0xc0677cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc020a924e0}, 0x660089, {0xc038a94900, {0x174301102bb82fa6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc06ffeec00, {0x63c85b8, 0xc020a924e0}, 0xc0571e8200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bb82fa6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bb82fa6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020a92450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020a92450}, {0xc0677cf3f8, 0x660089}, 0xc0677cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020a92450}, {0x203014}, 0xc0571e8000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020a923c0}, 0xc0571e8000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}, 0xc06e7cd760, 0xc02990d770) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}, 0x3, 0xc04eccad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0677cf980}}}, {0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}, 0xc06e7cd760, 0xc04eccad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020a923c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020a923c0}, {0x40, 0xc0677cfb70}, 0xc0677cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020a923c0}, {0x4e9d7a0, 0xc0571e8000}, 0xc06e7cd760, 0xc02990d770) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020a923c0}, 0xc06ba84360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc038a947e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc038a947e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799609 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02f71c000}, 0xc002d98cb0, {0x646cc48, 0xc02f71c060}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02f71c000}, 0xedb7a834e, 0xc0948cb320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02f71c000}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04f2a1020}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc002d98ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc04f2a1020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802262 [runnable]: sync.runtime_SemacquireMutex(0xc09981c690, 0x8f, 0x7fd57f2ba180) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc09981c538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc09981c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc08a923200, {0xffffffffffffff15, 0xc09981c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc08a923200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc08a923200, {0xc014797200, 0x32, 0x77}, {0xc02f6db600, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0934be398, {0xc014797200, 0x32, 0x77}, {0xc02f6db600, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0934bccc0, {0xc02f6db600, 0x74, 0x77}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0934bccc0, {0xc014797200, 0x74, 0xc02f6db600}, {0xc02f6db600, 0xc05a3020e0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0934bca00, {0xc02f6db600, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc00db479a8, {{0xc05a3020e0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0b58dc300, {{0xc043a684b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0833d2c00, {0x63c85b8, 0xc0aeef0120}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0aeef0120}, {0x64a6560, 0xc0b58dc300}, {0xc043a684b0, 0x676e75736d617362, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0aeef0120}, {0x6466ed8, 0xc00db47700}, {0xc043a684b0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc026662918}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04f1d7600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020edd800, {0x63c85b8, 0xc0aeef0120}, 0xc04f1d7600, 0xc0781188c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020edd800, {0x63c85b8, 0xc0aeef0120}, 0xc04f1d7600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0aeef0090}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0aeef0090}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0aeef0090}, {0xc00102e4e0}, 0xc04f1d7500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0aeef0030}, 0xc04f1d7500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0bb639530}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812173 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc019482c00}, 0xc0450fd9b0, 0x1, 0xc034fd90a8, 0x519bbc, 0xc09fd4ca80, 0xc09c9f54a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc019482c00}, 0x120, 0xc034fd9230, 0x9fd4ca20, 0x0, 0x660069, 0xc09fd4ca80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc019482c00}, 0xc09fd4ca20, {{{{0x0, 0x0}, {0xc063c79440, 0x30}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc019482c00}, 0x6352410, 0x3abf6340) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc019482c00}, {0xc09fd4c900, {0x174301107a113c4d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc019482c00}, 0xc02e2f35e0, 0xc034fd99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc019482c00}, 0x660089, {0xc09fd4c900, {0x174301107a113c4d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc019482c00}, 0xc0503f3e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a113c4d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a113c4d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc019482b70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019482b70}, {0xc034fdb3f8, 0x660089}, 0xc034fdb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc019482b70}, {0x7fe3488b7bc0}, 0xc0503f3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc019482ae0}, 0xc0503f3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}, 0xc01c022540, 0xc017f10f18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}, 0x3, 0xc06541d340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034fdb980}}}, {0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}, 0xc01c022540, 0xc06541d340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc019482ae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019482ae0}, {0x40, 0xc034fdbb70}, 0xc034fdbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc019482ae0}, {0x4e9d7a0, 0xc0503f3c00}, 0xc01c022540, 0xc017f10f18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc019482ae0}, 0xc02b523020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc02edc3440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc02edc3440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810226 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc07412afc0}, 0xc07231bcf8, 0x1, 0xc01a1ab0a8, 0x519bbc, 0xc07bd56780, 0xc0460e1740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0d06db8c0, {0x63c85b8, 0xc07412afc0}, 0xc05c772240, 0xc01a1ab1a8, 0x7bd56780, 0x0, 0x0, 0xc07bd56780, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc07412afc0}, 0xc07bd566c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc019561680, 0x9e}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07412afc0}, 0x6352260, 0x603b9ee0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc07412afc0}, {0xc07bd565a0, {0x174301102fcc5c7a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc07412afc0}, 0xc072e68aa0, 0xc01a1ab9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07412afc0}, 0x660089, {0xc07bd565a0, {0x174301102fcc5c7a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07412afc0}, 0xc06f0a6400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fcc5c7a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fcc5c7a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07412af30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07412af30}, {0xc01a1ad3f8, 0x660089}, 0xc01a1ad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07412af30}, {0x0}, 0xc06f0a6200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07412aea0}, 0xc06f0a6200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}, 0xc032c16920, 0xc073de48a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}, 0x3, 0xc062ededc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a1ad980}}}, {0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}, 0xc032c16920, 0xc062ededc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07412aea0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07412aea0}, {0x40, 0xc01a1adb70}, 0xc01a1adab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07412aea0}, {0x4e9d7a0, 0xc06f0a6200}, 0xc032c16920, 0xc073de48a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07412aea0}, 0xc027184060, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07bd56480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc07bd56480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807621 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc041376e10}, 0xc0997f9700, 0xc0b202b400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc041376e10}, 0xc0997f9700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058606be6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058606be6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041376d80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041376d80}, {0xc003cb53f8, 0x660089}, 0xc003cb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041376d80}, {0x3feffffffd645321}, 0xc0997f9500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041376cc0}, 0xc0997f9500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}, 0xc0866656a0, 0xc02385d890) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}, 0x3, 0xc061b678c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003cb5980}}}, {0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}, 0xc0866656a0, 0xc061b678c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041376cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041376cc0}, {0x40, 0xc003cb5b70}, 0xc003cb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041376cc0}, {0x4e9d7a0, 0xc0997f9500}, 0xc0866656a0, 0xc02385d890) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041376cc0}, 0xc02f8cdaa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0579f47e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0579f47e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812692 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802917 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc07cdb8300}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812668 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811771 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc075ecb3b0}, 0xc0a7338e28, 0x1, 0xc02688d0a8, 0x519bbc, 0xc02073d020, 0xc037eb4ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc075ecb3b0}, 0x120, 0xc02688d230, 0x2073cfc0, 0x0, 0x660069, 0xc02073d020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc075ecb3b0}, 0xc02073cfc0, {{{{0x0, 0x0}, {0xc0a432cd80, 0x11b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc075ecb3b0}, 0x6352410, 0x11fab3f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc075ecb3b0}, {0xc02073cea0, {0x174301107ace83a0, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc075ecb3b0}, 0xc023799720, 0xc02688d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc075ecb3b0}, 0x660089, {0xc02073cea0, {0x174301107ace83a0, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc075ecb3b0}, 0xc082920a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107ace83a0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107ace83a0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075ecb320}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075ecb320}, {0xc02688f3f8, 0x660089}, 0xc02688f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075ecb320}, {0x5f4f4e05460a0000}, 0xc082920800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc075ecb290}, 0xc082920800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}, 0xc029ae90e0, 0xc0208f0888) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}, 0x3, 0xc05aca0840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02688f980}}}, {0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}, 0xc029ae90e0, 0xc05aca0840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc075ecb290}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075ecb290}, {0x40, 0xc02688fb70}, 0xc02688fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc075ecb290}, {0x4e9d7a0, 0xc082920800}, 0xc029ae90e0, 0xc0208f0888) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc075ecb290}, 0xc05f712fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02073cd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc02073cd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806588 [IO wait]: internal/poll.runtime_pollWait(0x7fe456aa9100, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc07770d580, 0xc0503b8800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc07770d580, {0xc0503b8800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc07770d580, {0xc0503b8800, 0x735, 0xc070146500}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc025ab6320, {0xc0503b8800, 0x7eda79, 0xc0503b8805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc036d36180, {0xc0503b8800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc073a04450, {0xc0503b8800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc013d053f8, {0x631b900, 0xc073a04450}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc013d05180, {0x7fe42ab59310, 0xc036d36180}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc013d05180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc013d05180, {0xc09b1e5000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01cb7f110, {0xc09b1e5000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0aadde088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0aadde088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0aadde660, {0x634aac0, 0xc0aadde088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0aadde000, 0xc091aa3f60, 0xc004f4142b, 0xc004f4142d, {0x635bec0, 0xc01b6fc750}, 0xc000d6ae00, 0xc08e12e6a8, 0xc004f41429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0aadde000, {0x63c85b8, 0xc072086030}, 0xc091aa3f50, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc07d298038, 0x8}}, 0x0, 0xc0924c1fb0, 0xc072086000, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc072086030}, {0x643d810, 0xc036d36180}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc08dd7dc20}, {0x643d810, 0xc036d36180}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc08dd7dc20}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812357 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc023b90280, {0x63c85b8, 0xc046e1b050}, 0xc044b4a960, 0x1, 0xc03521f0a8, 0x519bbc, 0xc0140ac600, 0xc0748cc2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046e1b050}, 0x120, 0xc03521f230, 0x140ac5a0, 0x0, 0x660069, 0xc0140ac600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc023b90280, {0x63c85b8, 0xc046e1b050}, 0xc0140ac5a0, {{{{0x0, 0x0}, {0xc077e2fb00, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046e1b050}, 0x6352410, 0x54d3a2f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc023a88b90, {0x63c85b8, 0xc046e1b050}, {0xc0140ac360, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc023a88b90, {0x63c85b8, 0xc046e1b050}, 0xc07330eb40, 0xc03521f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046e1b050}, 0x660089, {0xc0140ac360, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023afc800, {0x63c85b8, 0xc046e1b050}, 0xc04ad1b900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046e1af90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046e1af90}, {0xc0352213f8, 0x660089}, 0xc035221400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046e1af90}, {0x7fe39445eca8}, 0xc08a8d3f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046e1ade0}, 0xc08a8d3f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}, 0xc06d7656e0, 0xc044b4b848) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}, 0x3, 0xc06e107a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc035221980}}}, {0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}, 0xc06d7656e0, 0xc06e107a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046e1ade0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046e1ade0}, {0x40, 0xc035221b70}, 0xc035221ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046e1ade0}, {0x4e9d7a0, 0xc08a8d3f00}, 0xc06d7656e0, 0xc044b4b848) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046e1ade0}, 0xc04c4d99e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0140ac240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0140ac240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807445 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0219b8000, {0x63c85b8, 0xc04a09b440}, 0xc08fa63400, 0xc052004c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc04a09b440}, 0xc08fa63400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105708d192, 0x0, 0x0}, 0xc09069ea10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105708d192, 0x0, 0x0}, 0xc09069ea10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04a09b3b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a09b3b0}, {0xc0056093f8, 0x660089}, 0xc005609400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04a09b3b0}, {0x3feffe1328e8ab1a}, 0xc08fa63200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04a09b320}, 0xc08fa63200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}, 0xc0455de1a0, 0xc04414f6c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}, 0x3, 0xc047e4f840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005609980}}}, {0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}, 0xc0455de1a0, 0xc047e4f840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04a09b320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04a09b320}, {0x40, 0xc005609b70}, 0xc005609ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04a09b320}, {0x4e9d7a0, 0xc08fa63200}, 0xc0455de1a0, 0xc04414f6c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04a09b320}, 0xc026f97f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc085361680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc085361680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801433 [runnable]: sync.runtime_SemacquireMutex(0xc097948690, 0x8f, 0x7fdf0d00d840) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc097948538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc097948560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc082688480, {0xffffffffffffff15, 0xc0979486c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc082688480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc082688480, {0xc04a0efcb0, 0x32, 0x8c}, {0xc07320db00, 0x32, 0x5f}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0870cb298, {0xc04a0efcb0, 0x32, 0x8c}, {0xc07320db00, 0x32, 0x5f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0870c9bc0, {0xc07320db00, 0x5f, 0x8c}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0870c9bc0, {0xc04a0efcb0, 0x5f, 0xc07320db00}, {0xc07320db00, 0xc052a1a0e0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0870c9900, {0xc07320db00, 0x32, 0x5f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0848ebe28, {{0xc052a1a0e0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc082cc5700, {{0xc0613b48d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc047e57400, {0x63c85b8, 0xc05154bb30}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc05154bb30}, {0x64a6560, 0xc082cc5700}, {0xc0613b48d0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc05154bb30}, {0x6466ed8, 0xc0848ebb80}, {0xc0613b48d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9acd8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc014ed1a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c93c00, {0x63c85b8, 0xc05154bb30}, 0xc014ed1a00, 0xc0465e8be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c93c00, {0x63c85b8, 0xc05154bb30}, 0xc014ed1a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05154baa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05154baa0}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05154baa0}, {0xc00102e4e0}, 0xc014ed1900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05154ba40}, 0xc014ed1900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02ef02480}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790553 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc05aa61ce0}, 0xc01f3f6cb0, {0x646cc48, 0xc05aa61d40}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc05aa61ce0}, 0xedb7a834e, 0xc059632120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc05aa61ce0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc042bbbb00}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01f3f6ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc042bbbb00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219141848 [select, 1 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc022ae70c0) google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:112 +0x79 created by google.golang.org/grpc.newCCBalancerWrapper google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:73 +0x22f goroutine 219347285 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea2d70, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06f54ed80, 0xc04a7a5800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06f54ed80, {0xc04a7a5800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06f54ed80, {0xc04a7a5800, 0x735, 0xc03821d7c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc04f454328, {0xc04a7a5800, 0x7eda79, 0xc04a7a5805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0728592c0, {0xc04a7a5800, 0x1, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc048c9b608, {0xc04a7a5800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc05992c5f8, {0x631b900, 0xc048c9b608}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc05992c380, {0x7fe42ab59310, 0xc0728592c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc05992c380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc05992c380, {0xc049e6e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0496f9158, {0xc049e6e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc040203588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc040203588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc040203b60, {0x634aac0, 0xc040203588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc040203500, 0xc04a3d3a60, 0xc06d9f142b, 0xc06d9f142d, {0x635bec0, 0xc01808db90}, 0xc000d6ae00, 0xc04f6a06e8, 0xc06d9f1429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc040203500, {0x63c85b8, 0xc03cc97770}, 0xc04a3d3a50, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc064ebf038, 0x8}}, 0x0, 0xc03cc97710, 0xc03cc97740, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc03cc97770}, {0x643d810, 0xc0728592c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc02b87be90}, {0x643d810, 0xc0728592c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc02b87be90}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802871 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0577b5050) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0577b50b8, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0952d31f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc050df1880, {0xc020f34400, 0x2b, 0x79}, {0xc032d06a50, 0x2b, 0x2f}, 0x1, 0x31) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0952d3088, {0xc020f34400, 0x1, 0x2f}, {0xc032d06a50, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0952d3088, {0xc020f34400, 0x2b, 0x79}, {0xc032d06a50, 0x2b, 0x2f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0952d27c0, {0xc032d06a50, 0x2f, 0x79}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0952d27c0, {0xc020f34400, 0x2b, 0xc032d06a50}, {0xc032d06a50, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0952d2500, {0xc032d06a50, 0x2b, 0x2f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc065ddb608, {{0xc036d42b40, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0728cae00, {{0xc036d42b40, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0529abc00, {0x63c85b8, 0xc014e11290}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc014e11290}, {0x64a6560, 0xc0728cae00}, {0xc036d42b40, 0xc00bc1fa40, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc014e11290}, {0x6466ed8, 0xc065ddb600}, {0xc036d42b40, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0770d9050}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0489b2b00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020edcc00, {0x63c85b8, 0xc014e11290}, 0xc0489b2b00, 0xc04724b0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020edcc00, {0x63c85b8, 0xc014e11290}, 0xc0489b2b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc014e10f00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc014e10f00}, {0xc0577bb3f8, 0x660089}, 0xc0577bb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc014e10f00}, {0x6574617669727065}, 0xc0489b2600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc014e10ba0}, 0xc0489b2600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}, 0xc021d4cc20, 0xc0183b24e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}, 0x3, 0xc07136a580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0577bb980}}}, {0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}, 0xc021d4cc20, 0xc07136a580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc014e10ba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc014e10ba0}, {0x40, 0xc0577bbb70}, 0xc0577bbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc014e10ba0}, {0x4e9d7a0, 0xc0489b2600}, 0xc021d4cc20, 0xc0183b24e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc014e10ba0}, 0xc02c8cfb60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc046c96d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc046c96d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809745 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f6e83c0, {0x63c85b8, 0xc04ef328d0}, 0xc08871c3d8, 0x1, 0xc0220730a8, 0x519bbc, 0xc03f8bcf60, 0xc04012df20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05a483b00, {0x63c85b8, 0xc04ef328d0}, 0xc03f8bcf60, 0xc0220731a8, 0x31, 0x40, 0x0, 0xc03f8bcf60, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f6e83c0, {0x63c85b8, 0xc04ef328d0}, 0xc03f8bcea0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc078a41440, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ef328d0}, 0x6352260, 0x300d1cb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f6dc320, {0x63c85b8, 0xc04ef328d0}, {0xc03f8bcd80, {0x174301102e4959b8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f6dc320, {0x63c85b8, 0xc04ef328d0}, 0xc05278a280, 0xc0220739f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ef328d0}, 0x660089, {0xc03f8bcd80, {0x174301102e4959b8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f6ec400, {0x63c85b8, 0xc04ef328d0}, 0xc0601c9900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e4959b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e4959b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ef327e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef327e0}, {0xc0220753f8, 0x660089}, 0xc022075400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ef327e0}, {0x0}, 0xc0601c9700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ef32750}, 0xc0601c9700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}, 0xc0772c34a0, 0xc08871d9f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}, 0x3, 0xc011471600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc022075980}}}, {0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}, 0xc0772c34a0, 0xc011471600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ef32750}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef32750}, {0x40, 0xc022075b70}, 0xc022075ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ef32750}, {0x4e9d7a0, 0xc0601c9700}, 0xc0772c34a0, 0xc08871d9f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ef32750}, 0xc02e64fda0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03f8bcc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03f8bcc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809488 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020f88b40, {0x63c85b8, 0xc07069ccc0}, 0xc049d48870, 0x1, 0xc019c1f0a8, 0x519bbc, 0xc05fc92240, 0xc06ed4cba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc07069ccc0}, 0x1e0, 0xc019c1f230, 0x5fc921e0, 0x0, 0x660069, 0xc05fc92240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020f88b40, {0x63c85b8, 0xc07069ccc0}, 0xc05fc921e0, {{{{0x0, 0x0}, {0xc040edf0e0, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc07069ccc0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021106230, {0x63c85b8, 0xc07069ccc0}, {0xc07479bd40, {0x174301106409cf95, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021106230, {0x63c85b8, 0xc07069ccc0}, 0xc039b4a3c0, 0xc019c1f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07069ccc0}, 0x660089, {0xc07479bd40, {0x174301106409cf95, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c1c00, {0x63c85b8, 0xc07069ccc0}, 0xc09a39b600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106409cf95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106409cf95, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07069cbd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07069cbd0}, {0xc019c213f8, 0x660089}, 0xc019c21400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07069cbd0}, {0xf4619234061fea62}, 0xc09a39b300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07069cb40}, 0xc09a39b300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}, 0xc03329fbe0, 0xc0533946c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}, 0x3, 0xc06fb55e00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc019c21980}}}, {0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}, 0xc03329fbe0, 0xc06fb55e00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07069cb40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07069cb40}, {0x40, 0xc019c21b70}, 0xc019c21ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07069cb40}, {0x4e9d7a0, 0xc09a39b300}, 0xc03329fbe0, 0xc0533946c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07069cb40}, 0xc0271c4360, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc087816240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc087816240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803974 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016296c80, {0x63c85b8, 0xc07e1bb170}, 0xc02a7aafc0, 0x1, 0xc0744d30a8, 0x519bbc, 0xc052a882a0, 0xc04eded800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07e1bb170}, 0x120, 0xc0744d3230, 0x52a88240, 0x0, 0x660069, 0xc052a882a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016296c80, {0x63c85b8, 0xc07e1bb170}, 0xc052a88240, {{{{0x0, 0x0}, {0xc05bf36000, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07e1bb170}, 0x6352410, 0x55acfc00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc008447130, {0x63c85b8, 0xc07e1bb170}, {0xc052a88120, {0x174301103094392a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc008447130, {0x63c85b8, 0xc07e1bb170}, 0xc05f31dea0, 0xc0744d39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07e1bb170}, 0x660089, {0xc052a88120, {0x174301103094392a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000df6400, {0x63c85b8, 0xc07e1bb170}, 0xc097368b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103094392a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07e1bb0b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e1bb0b0}, {0xc0744d53f8, 0x660089}, 0xc0744d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07e1bb0b0}, {0xc0015221e0}, 0xc097368800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07e1bb020}, 0xc097368800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}, 0xc0606eb7c0, 0xc038313fc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}, 0x3, 0xc02661cd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0744d5980}}}, {0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}, 0xc0606eb7c0, 0xc02661cd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07e1bb020}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e1bb020}, {0x40, 0xc0744d5b70}, 0xc0744d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07e1bb020}, {0x4e9d7a0, 0xc097368800}, 0xc0606eb7c0, 0xc038313fc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07e1bb020}, 0xc0362930e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050cff9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc050cff9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809308 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04da8f800}, 0xc035f70720, 0x1, 0xc0116c90a8, 0x519bbc, 0xc06ae3c060, 0xc09b2747e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04da8f800}, 0x1e0, 0xc0116c9230, 0x6ae3c000, 0x0, 0x660069, 0xc06ae3c060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04da8f800}, 0xc06ae3c000, {{{{0x0, 0x0}, {0xc048361d40, 0xd7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04da8f800}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04da8f800}, {0xc079c18a20, {0x174301106496180a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04da8f800}, 0xc066131d60, 0xc0116c99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04da8f800}, 0x660089, {0xc079c18a20, {0x174301106496180a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04da8f800}, 0xc04eb50100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106496180a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106496180a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04da8f770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04da8f770}, {0xc0116cb3f8, 0x660089}, 0xc0116cb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04da8f770}, {0x7fe3c01976a0}, 0xc09d60bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04da8f6e0}, 0xc09d60bf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}, 0xc028ad4ca0, 0xc0452214e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}, 0x3, 0xc0499b5280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0116cb980}}}, {0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}, 0xc028ad4ca0, 0xc0499b5280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04da8f6e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04da8f6e0}, {0x40, 0xc0116cbb70}, 0xc0116cbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04da8f6e0}, {0x4e9d7a0, 0xc09d60bf00}, 0xc028ad4ca0, 0xc0452214e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04da8f6e0}, 0xc047b42840, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc079c18900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc079c18900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799213 [runnable]: sync.runtime_SemacquireMutex(0xc0922e6690, 0x8f, 0x7fda6673db00) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc0922e6538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0922e6560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc01260a900, {0xffffffffffffff15, 0xc0922e66c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc01260a900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc01260a900, {0xc05c6848c0, 0x32, 0xa0}, {0xc03f1a5080, 0x32, 0x52}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc085f76398, {0xc05c6848c0, 0x32, 0xa0}, {0xc03f1a5080, 0x32, 0x52}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc085f74cc0, {0xc03f1a5080, 0x52, 0xa0}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc085f74cc0, {0xc05c6848c0, 0x52, 0xc03f1a5080}, {0xc03f1a5080, 0xc03c45efc0, 0x31}, 0x84) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc085f74a00, {0xc03f1a5080, 0x32, 0x52}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc07716aea8, {{0xc03c45efc0, 0x31, 0x84}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04d14aa00, {{0xc05b588180, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0678a1400, {0x63c85b8, 0xc0b8889290}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0b8889290}, {0x64a6560, 0xc04d14aa00}, {0xc05b588180, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0b8889290}, {0x6466ed8, 0xc07716ac00}, {0xc05b588180, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc03787e360}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05059f600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0207d6400, {0x63c85b8, 0xc0b8889290}, 0xc05059f600, 0xc022383a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0207d6400, {0x63c85b8, 0xc0b8889290}, 0xc05059f600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0b8889200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0b8889200}, {0xc0ad83c8f0, 0x660089}, 0xc0ad83c8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0b8889200}, {0xc00102e4e0}, 0xc05059f500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0b88891a0}, 0xc05059f500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03cc78cc0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219490571 [IO wait]: internal/poll.runtime_pollWait(0x7fdb124bcc70, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc02014e600, 0xc03cdc9800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02014e600, {0xc03cdc9800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc02014e600, {0xc03cdc9800, 0x735, 0xc0388563c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc034bdbff8, {0xc03cdc9800, 0x7eda79, 0xc03cdc9805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc053f868a0, {0xc03cdc9800, 0x46a86f, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc070822a50, {0xc03cdc9800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0ce4b4cf8, {0x631b900, 0xc070822a50}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0ce4b4a80, {0x7fe42ab59310, 0xc053f868a0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0ce4b4a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0ce4b4a80, {0xc025193000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc026fa9428, {0xc025193000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0b5310088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0b5310088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0b5310660, {0x634aac0, 0xc0b5310088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0b5310000, 0xc02b7b1570, 0xc09d09b42b, 0xc09d09b42d, {0x635bec0, 0xc03ef1ecf0}, 0xc000d6ae00, 0xc02cec5208, 0xc09d09b429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0b5310000, {0x63c85b8, 0xc043972840}, 0xc02b7b1520, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc01960701e, 0x8}}, 0x0, 0xc0439727e0, 0xc043972810, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc043972840}, {0x643d810, 0xc053f868a0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0473c4f60}, {0x643d810, 0xc053f868a0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0473c4f60}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219788187 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc063ef4e10, 0x90) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0a0fa2a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0a0fa30e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0400e0000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0400e0000, {0x63c8510, 0xc063ef4e40}, 0xc053b77dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02d739670) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc063ef4e40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02d739670) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810120 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0181f8c80, {0x63c85b8, 0xc066400f00}, 0xc063a51ea8, 0x1, 0xc0063690a8, 0x519bbc, 0xc052a11100, 0xc03a58cd20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc033f8ea20, {0x63c85b8, 0xc066400f00}, 0xc052a11100, 0xc0063691a8, 0x30, 0x40, 0x0, 0xc052a11100, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0181f8c80, {0x63c85b8, 0xc066400f00}, 0xc052a10e00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0727e6120, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc066400f00}, 0x63521a0, 0x50b34280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0181de7d0, {0x63c85b8, 0xc066400f00}, {0xc0a1690d80, {0x1743010ff14f83ff, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0181de7d0, {0x63c85b8, 0xc066400f00}, 0xc01bce3f40, 0xc0063699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066400f00}, 0x660089, {0xc0a1690d80, {0x1743010ff14f83ff, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0182b0800, {0x63c85b8, 0xc066400f00}, 0xc051d52400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff14f83ff, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff14f83ff, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066400e70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066400e70}, {0xc00636b3f8, 0x660089}, 0xc00636b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066400e70}, {0x7fe34e287968}, 0xc051d52100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066400d50}, 0xc051d52100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}, 0xc0198daa40, 0xc0462b4ff0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}, 0x3, 0xc074746940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00636b980}}}, {0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}, 0xc0198daa40, 0xc074746940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066400d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066400d50}, {0x40, 0xc00636bb70}, 0xc00636bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066400d50}, {0x4e9d7a0, 0xc051d52100}, 0xc0198daa40, 0xc0462b4ff0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066400d50}, 0xc03d331b60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0a1690c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0a1690c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791442 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc036745f20}, 0xc032b20cb0, {0x646cc48, 0xc036745f80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc036745f20}, 0xedb7a834e, 0xc085a18120) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc036745f20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc049628d80}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc032b20ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc049628d80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812703 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802517 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc0619a7b60}, 0xc063a51398, 0x1, 0xc0219e30a8, 0x519bbc, 0xc039009e60, 0xc039e629c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0619a7b60}, 0x1e0, 0xc0219e3230, 0x39009e00, 0x0, 0x660069, 0xc039009e60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc0619a7b60}, 0xc039009e00, {{{{0x0, 0x0}, {0xc04e1c9200, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0619a7b60}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc0619a7b60}, {0xc0359a8b40, {0x1743011021918655, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc0619a7b60}, 0xc03c3eedc0, 0xc0219e39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0619a7b60}, 0x660089, {0xc0359a8b40, {0x1743011021918655, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc0619a7b60}, 0xc00137b800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011021918655, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011021918655, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0619a7ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0619a7ad0}, {0xc0219e53f8, 0x660089}, 0xc0219e5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0619a7ad0}, {0xc01b312618}, 0xc00137b600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0619a7a40}, 0xc00137b600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}, 0xc06c86be20, 0xc0440f6480) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}, 0x3, 0xc06d6c6d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0219e5980}}}, {0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}, 0xc06c86be20, 0xc06d6c6d40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0619a7a40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0619a7a40}, {0x40, 0xc0219e5b70}, 0xc0219e5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0619a7a40}, {0x4e9d7a0, 0xc00137b600}, 0xc06c86be20, 0xc0440f6480) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0619a7a40}, 0xc06dffd9e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0359a8a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0359a8a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807402 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc025cd4400, {0x63c85b8, 0xc02c73cc30}, 0xc092036900, 0xc0742ea820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025cd4400, {0x63c85b8, 0xc02c73cc30}, 0xc092036900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011056f83dd5, 0x0, 0x0}, 0xc0907f4390, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011056f83dd5, 0x0, 0x0}, 0xc0907f4390, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02c73cb10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c73cb10}, {0xc0052293f8, 0x660089}, 0xc005229400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02c73cb10}, {0xc0015221e0}, 0xc092036700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02c73c960}, 0xc092036700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}, 0xc007b4e8a0, 0xc046971530) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}, 0x3, 0xc039e7adc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005229980}}}, {0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}, 0xc007b4e8a0, 0xc039e7adc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02c73c960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c73c960}, {0x40, 0xc005229b70}, 0xc005229ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02c73c960}, {0x4e9d7a0, 0xc092036700}, 0xc007b4e8a0, 0xc046971530) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02c73c960}, 0xc05205f3e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081fd5440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081fd5440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800548 [runnable]: syscall.Syscall6(0x11, 0x142e, 0x7fe1c771b020, 0x36ba, 0x1e57e55, 0x0, 0x0) GOROOT/src/syscall/asm_linux_amd64.s:43 +0x5 syscall.Pread(0x2, {0x7fe1c771b020, 0x118564e, 0xc00074aee0}, 0x0) GOROOT/src/syscall/zsyscall_linux_amd64.go:1212 +0x58 internal/poll.(*FD).Pread(0xc04dde1ba0, {0x7fe1c771b020, 0x4ba25b, 0x36ba}, 0x200000003) GOROOT/src/internal/poll/fd_unix.go:193 +0xf9 os.(*File).pread(...) GOROOT/src/os/file_posix.go:41 os.(*File).ReadAt(0xc03f55a3f0, {0x7fe1c771b020, 0xc095e743c0, 0x1184134}, 0x36da) GOROOT/src/os/file.go:137 +0x191 github.com/cockroachdb/pebble/vfs.enospcFile.ReadAt(...) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_full.go:361 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc011f09080, {0x1e57e55, 0x36b5}, 0x0, 0xc042170ff8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2339 +0x3f2 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc042170d80, {0xffffffffffffff15, 0xc095e745c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc042170d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc042170d80, {0xc0227fbce0, 0x2b, 0x6d}, 0x13, 0x1e) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc042170d80, {0xc024f57f80, 0x2b, 0x6d}, {0xc0227fbce0, 0x51c9860, 0xc0227fbce0}, 0x2b, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc042170d80, {0xc024f57f80, 0x2b, 0x77}, {0xc0227fbce0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091ed4d98, {0xc024f57f80, 0x2b, 0x77}, {0xc0227fbce0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091ed36c0, {0xc0227fbce0, 0x6d, 0x77}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091ed36c0, {0xc024f57f80, 0x2b, 0xc0227fbce0}, {0xc0227fbce0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091ed3400, {0xc0227fbce0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0514ad188, {{0xc0650f1710, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0728ca300, {{0xc0650f1710, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04fee0000, {0x63c85b8, 0xc01d02d7d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01d02d7d0}, {0x64a6560, 0xc0728ca300}, {0xc0650f1710, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01d02d7d0}, {0x6466ed8, 0xc0514ad180}, {0xc0650f1710, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc07d48fab8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04da84000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020aef800, {0x63c85b8, 0xc01d02d7d0}, 0xc04da84000, 0xc04cd350e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020aef800, {0x63c85b8, 0xc01d02d7d0}, 0xc04da84000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01d02d740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d02d740}, {0xc0d68328f0, 0x660089}, 0xc0d68328f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01d02d740}, {0xc00102e4e0}, 0xc06adf9e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01d02d6e0}, 0xc06adf9e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7ba70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808269 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc066821140}, 0xc0715c98c0, 0x1, 0x0, 0x1, 0xc075bc3c80, 0xc04b9d5380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc066821140}, 0x120, 0xc00b75a728, 0x75bc3c20, 0x0, 0x660069, 0xc075bc3c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc066821140}, 0xc075bc3c20, {{{{0x0, 0x0}, {0xc0434b4ea0, 0x1d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc066821140}, 0x6352410, 0x2ad6f508) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc066821140}, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc066821140}, 0xc02797d860, 0xc07c63eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066821140}, 0x660089, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc066821140}, 0xc09c379100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0668210b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0668210b0}, {0xc0025148f0, 0x660089}, 0xc0025148f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0668210b0}, {0xc00102e4e0}, 0xc09c379000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066821050}, 0xc09c379000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc045e73a70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812581 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811893 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6000, {0x63c85b8, 0xc05327fa40}, 0xc021241818, 0x1, 0x0, 0x1, 0xc06e7d0060, 0xc085b9f380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05327fa40}, 0x120, 0xc033b26728, 0x6e7d0000, 0x0, 0x660069, 0xc06e7d0060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6000, {0x63c85b8, 0xc05327fa40}, 0xc06e7d0000, {{{{0x0, 0x0}, {0xc05210d0e0, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05327fa40}, 0x6352410, 0x33a922c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddb30, {0x63c85b8, 0xc05327fa40}, {0xc04305de60, {0x174301107bade4d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc05327fa40}, 0xc041b80500, 0xc058160ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05327fa40}, 0x660089, {0xc04305de60, {0x174301107bade4d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc05327fa40}, 0xc04c24a800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107bade4d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107bade4d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05327f9b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05327f9b0}, {0xc003eee8f0, 0x660089}, 0xc003eee8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05327f9b0}, {0xc00102e4e0}, 0xc04c24a700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05327f950}, 0xc04c24a700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc05327f620}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809791 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc07e09b050}, 0xc05b94f380, 0x1, 0xc006cab0a8, 0x519bbc, 0xc081cee1e0, 0xc036e480c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc016b52880, {0x63c85b8, 0xc07e09b050}, 0xc04d22f680, 0xc006cab1a8, 0x81cee1e0, 0x0, 0x0, 0xc081cee1e0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc07e09b050}, 0xc081cee120, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0520c1200, 0x4e}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07e09b050}, 0x6352260, 0x29b7baf0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc07e09b050}, {0xc081cee000, {0x174301102bd75419, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc07e09b050}, 0xc050934820, 0xc006cab9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07e09b050}, 0x660089, {0xc081cee000, {0x174301102bd75419, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc07e09b050}, 0xc0bd5b8800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102bd75419, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102bd75419, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07e09afc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e09afc0}, {0xc006cad3f8, 0x660089}, 0xc006cad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07e09afc0}, {0x0}, 0xc0bd5b8500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07e09af30}, 0xc0bd5b8500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}, 0xc050031100, 0xc0292d4468) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}, 0x3, 0xc06fb0c2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006cad980}}}, {0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}, 0xc050031100, 0xc06fb0c2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07e09af30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e09af30}, {0x40, 0xc006cadb70}, 0xc006cadab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07e09af30}, {0x4e9d7a0, 0xc0bd5b8500}, 0xc050031100, 0xc0292d4468) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07e09af30}, 0xc046506420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc046ed6ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc046ed6ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808238 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc046f341b0}, 0xc039e4f100, 0xc0468857c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc046f341b0}, 0xc039e4f100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105d4dee7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105d4dee7b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046f34120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046f34120}, {0xc0051e13f8, 0x660089}, 0xc0051e1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046f34120}, {0x408b27fc4123cf2b}, 0xc039e4ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046f34000}, 0xc039e4ed00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}, 0xc0748d5a80, 0xc0198ab0b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}, 0x3, 0xc04e3540c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0051e1980}}}, {0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}, 0xc0748d5a80, 0xc04e3540c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046f34000}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046f34000}, {0x40, 0xc0051e1b70}, 0xc0051e1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046f34000}, {0x4e9d7a0, 0xc039e4ed00}, 0xc0748d5a80, 0xc0198ab0b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046f34000}, 0xc04acae5a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c350ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c350ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809718 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0720b9e30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810098 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e035680, {0x63c85b8, 0xc043691e00}, 0xc0303429d8, 0x1, 0x70801806ebc7ca, 0x37e11d600, 0xc062b87aa0, 0xc04141ff20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05b1e0900, {0x63c85b8, 0xc043691e00}, 0xc062b87aa0, 0xc014a917d8, 0x41, 0x60, 0x3, 0xc062b87aa0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e035680, {0x63c85b8, 0xc043691e00}, 0xc062b879e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0166c9440, 0x9}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc043691e00}, 0x6352260, 0x300d1f30) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd6d0, {0x63c85b8, 0xc043691e00}, {0xc062b878c0, {0x174301102e67b889, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd6d0, {0x63c85b8, 0xc043691e00}, 0xc0688e8460, 0xc014a92020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc043691e00}, 0x660089, {0xc062b878c0, {0x174301102e67b889, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc043691e00}, 0xc06eeec100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102e67b889, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102e67b889, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043691d70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043691d70}, {0xc07dd21a28, 0x660089}, 0xc07dd21a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043691d70}, {0x24}, 0xc06eeec000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043691d10}, 0xc06eeec000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc04aab2cd0, {0x63c85b8, 0xc043691c80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc04aab2cd0, {0x63c85b8, 0xc043691c80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc043691c80}, {0x4f67ad2, 0x6446ff8}, 0xc001f12fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219802459 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0300a9c00, {0x63c85b8, 0xc0c78b52f0}, 0xc0a7740f00, 0xc07ae3a280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0300a9c00, {0x63c85b8, 0xc0c78b52f0}, 0xc0a7740f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011023a3e20c, 0x0, 0x0}, 0xc04fc36910, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011023a3e20c, 0x0, 0x0}, 0xc04fc36910, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0c78b5260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0c78b5260}, {0xc073095918, 0x660089}, 0xc073095920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0c78b5260}, {0xc00102e4e0}, 0xc0a7740e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0c78b5200}, 0xc0a7740e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xa, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc05c82b8c0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0c78b5170}, {0xc00dfd36b0, 0x24}, 0x4823301a1, 0xc016472d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc0d3a35e38, 0xc0781447c8}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0c78b5170}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812004 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc03298ce70}, 0xc063fae000, 0x1, 0x70801819369e5b, 0x37e11d600, 0xc06e7d1380, 0xc059510a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03298ce70}, 0x120, 0xc0346cb230, 0x6e7d1320, 0x0, 0x660069, 0xc06e7d1380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc03298ce70}, 0xc06e7d1320, {{{{0x0, 0x0}, {0xc0457bb200, 0x64}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03298ce70}, 0x6352410, 0x6b066670) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc03298ce70}, {0xc06e7d1200, {0x174301107c36161b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc03298ce70}, 0xc05efdea00, 0xc0346cb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03298ce70}, 0x660089, {0xc06e7d1200, {0x174301107c36161b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc03298ce70}, 0xc04c24b200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107c36161b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107c36161b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03298ccc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03298ccc0}, {0xc0346cd3f8, 0x660089}, 0xc0346cd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03298ccc0}, {0xc0015221e0}, 0xc04c24b000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03298cb10}, 0xc04c24b000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}, 0xc08d07b480, 0xc021241fc8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}, 0x3, 0xc0a43dc8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0346cd980}}}, {0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}, 0xc08d07b480, 0xc0a43dc8c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03298cb10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03298cb10}, {0x40, 0xc0346cdb70}, 0xc0346cdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03298cb10}, {0x4e9d7a0, 0xc04c24b000}, 0xc08d07b480, 0xc021241fc8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03298cb10}, 0xc0cc0036e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc065f5d0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc065f5d0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802032 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02072e800, {0x63c85b8, 0xc0641a3ef0}, 0xc061fff800, 0xc045b0aa00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02072e800, {0x63c85b8, 0xc0641a3ef0}, 0xc061fff800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110173aa8b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110173aa8b8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0641a3e60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0641a3e60}, {0xc01322b3f8, 0x660089}, 0xc01322b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0641a3e60}, {0x7fe3952dc998}, 0xc061fff400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0641a3dd0}, 0xc061fff400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}, 0xc04ecda9c0, 0xc0302f5668) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}, 0x3, 0xc06e1c6640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01322b980}}}, {0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}, 0xc04ecda9c0, 0xc06e1c6640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0641a3dd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0641a3dd0}, {0x40, 0xc01322bb70}, 0xc01322bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0641a3dd0}, {0x4e9d7a0, 0xc061fff400}, 0xc04ecda9c0, 0xc0302f5668) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0641a3dd0}, 0xc04ad081e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037fe19e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc037fe19e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808063 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021d552c0, {0x63c85b8, 0xc06e747e30}, 0xc041f29968, 0x1, 0xc0058750a8, 0x519bbc, 0xc0731b5920, 0xc07614af00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06e747e30}, 0x120, 0xc005875230, 0x731b58c0, 0x0, 0x660069, 0xc0731b5920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021d552c0, {0x63c85b8, 0xc06e747e30}, 0xc0731b58c0, {{{{0x0, 0x0}, {0xc01f596480, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06e747e30}, 0x6352410, 0x13b80b90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3cbe0, {0x63c85b8, 0xc06e747e30}, {0xc0731b57a0, {0x174301105a865aee, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3cbe0, {0x63c85b8, 0xc06e747e30}, 0xc07229e0a0, 0xc0058759f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06e747e30}, 0x660089, {0xc0731b57a0, {0x174301105a865aee, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021daa000, {0x63c85b8, 0xc06e747e30}, 0xc0326ee900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a865aee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a865aee, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e747da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e747da0}, {0xc0058773f8, 0x660089}, 0xc005877400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e747da0}, {0x7fe423f5f600}, 0xc0326ee700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e747d10}, 0xc0326ee700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}, 0xc0716b5a00, 0xc05a339a28) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}, 0x3, 0xc06dd55a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005877980}}}, {0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}, 0xc0716b5a00, 0xc06dd55a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06e747d10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e747d10}, {0x40, 0xc005877b70}, 0xc005877ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06e747d10}, {0x4e9d7a0, 0xc0326ee700}, 0xc0716b5a00, 0xc05a339a28) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06e747d10}, 0xc0cd848720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc079ab0c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc079ab0c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804965 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016d6d2c0, {0x63c85b8, 0xc02cf04660}, 0xc041854a20, 0x1, 0xc06ff810a8, 0x519bbc, 0xc06f6b8600, 0xc06e5d3a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02cf04660}, 0x4bf403, 0xc06ff81230, 0x6f6b8600, 0xc06f6b85a0, 0x18, 0xc06f6b8600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016d6d2c0, {0x63c85b8, 0xc02cf04660}, 0xc06f6b85a0, {{{{0x0, 0x0}, {0xc0604927e0, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02cf04660}, 0x6352410, 0x2f62fcb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce3180, {0x63c85b8, 0xc02cf04660}, {0xc06f6b8480, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce3180, {0x63c85b8, 0xc02cf04660}, 0xc059ef7540, 0xc06ff819f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02cf04660}, 0x660089, {0xc06f6b8480, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016efac00, {0x63c85b8, 0xc02cf04660}, 0xc0448c2d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02cf045d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02cf045d0}, {0xc06ff833f8, 0x660089}, 0xc06ff83400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02cf045d0}, {0x7fe42b763e78}, 0xc0448c2a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02cf044b0}, 0xc0448c2a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}, 0xc0664fece0, 0xc041855ae8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}, 0x3, 0xc054bb84c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06ff83980}}}, {0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}, 0xc0664fece0, 0xc054bb84c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02cf044b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02cf044b0}, {0x40, 0xc06ff83b70}, 0xc06ff83ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}, 0x203015, 0x203015) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02cf044b0}, {0x4e9d7a0, 0xc0448c2a00}, 0xc0664fece0, 0xc041855ae8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02cf044b0}, 0xc04977fbc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06f5abe60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06f5abe60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802915 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f632a00, {0x63c85b8, 0xc063eadc80}, 0xc028a61518, 0x1, 0xc055c0f0a8, 0x519bbc, 0xc040e04180, 0xc0a2f151e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc063eadc80}, 0x120, 0xc055c0f230, 0x40e04120, 0x0, 0x660069, 0xc040e04180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f632a00, {0x63c85b8, 0xc063eadc80}, 0xc040e04120, {{{{0x0, 0x0}, {0xc079381560, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc063eadc80}, 0x6352230, 0x49a42228) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f583ea0, {0x63c85b8, 0xc063eadc80}, {0xc040e04000, {0x1743011026cd5132, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f583ea0, {0x63c85b8, 0xc063eadc80}, 0xc02f2c7cc0, 0xc055c0f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc063eadc80}, 0x660089, {0xc040e04000, {0x1743011026cd5132, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f658000, {0x63c85b8, 0xc063eadc80}, 0xc08310be00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011026cd5132, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011026cd5132, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc063eadbf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063eadbf0}, {0xc055c113f8, 0x660089}, 0xc055c11400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc063eadbf0}, {0x100a5612580a17a1}, 0xc08310ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc063eadb60}, 0xc08310ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}, 0xc0062f46e0, 0xc0218a6678) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}, 0x3, 0xc051ef5340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055c11980}}}, {0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}, 0xc0062f46e0, 0xc051ef5340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc063eadb60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063eadb60}, {0x40, 0xc055c11b70}, 0xc055c11ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc063eadb60}, {0x4e9d7a0, 0xc08310ba00}, 0xc0062f46e0, 0xc0218a6678) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc063eadb60}, 0xc0869724e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0764a5d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0764a5d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810904 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc017f14640, {0x63c85b8, 0xc01c5488a0}, 0xc023238cc0, 0x1, 0xc000f43670, 0x519bbc, 0xc0ce793b80, 0xc063c1a780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc029d60480, {0x63c85b8, 0xc01c5488a0}, 0xc0ce793b80, 0xc000f43770, 0x30, 0x40, 0x0, 0xc0ce793b80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc017f14640, {0x63c85b8, 0xc01c5488a0}, 0xc02c7d7f80, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc03911a5a0, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01c5488a0}, 0x63521a0, 0x51b897b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc017e6e550, {0x63c85b8, 0xc01c5488a0}, {0xc00d2f6fc0, {0x174301100bbabe61, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc017e6e550, {0x63c85b8, 0xc01c5488a0}, 0xc01c697b80, 0xc000f43fb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01c5488a0}, 0x660089, {0xc00d2f6fc0, {0x174301100bbabe61, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017f21400, {0x63c85b8, 0xc01c5488a0}, 0xc059270700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100bbabe61, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100bbabe61, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c548720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c548720}, {0xc081e5d9c0, 0x660089}, 0xc081e5d9c8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c548720}, {0xc00102e4e0}, 0xc059270600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c5485d0}, 0xc059270600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x63c85b8, 0xc044d039b0}, {0x7fe422a51230, 0xc06194d680}, 0xc020ad0ea0, {0xc04a489b00, 0x6, 0x6}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x24d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x63c85b8, 0xc044d039b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219753710 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0679d7600, {0x63c85b8, 0xc08d3c1b60}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc04cac6600, {0x63c85b8, 0xc08d3c1b60}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc04cac6600, {0x63c8510, 0xc08a39cc00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc04cac6600, {0x63c8510, 0xc08a39cc00}, {0x63f7db8, 0x9bf6f48}, 0xc096cb5088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc04cac6600, {0x63c85b8, 0xc088fd7d10}, {{0x63f7db8, 0x9bf6f48}, {0xc035d9224c, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc04cac6600, {0x63c85b8, 0xc088fd7d10}, {{0x63f7db8, 0x9bf6f48}, {0xc035d9224c, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc035d9224c, 0x6}, 0x0, 0x0}, {0x34a388b1, 0xedb7a834f, 0x0}, {0x34a39236, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04cac6600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04cac6600, {0x63c8510, 0xc0b36c3cc0}, 0xc06dd99dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08c0ffa30) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0b36c3cc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08c0ffa30) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801182 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc09668a548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc09668a5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0931680f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc03fb9c380, {0xc030726600, 0x2b, 0x74}, {0xc0837620e0, 0x2b, 0x6d}, 0x1, 0x80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc093167f88, {0xc030726600, 0x1, 0x6d}, {0xc0837620e0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093167f88, {0xc030726600, 0x2b, 0x74}, {0xc0837620e0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0931676c0, {0xc0837620e0, 0x6d, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0931676c0, {0xc030726600, 0x2b, 0xc0837620e0}, {0xc0837620e0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093167400, {0xc0837620e0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05a2da688, {{0xc03f653e90, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04830be00, {{0xc03f653e90, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0359b8000, {0x63c85b8, 0xc0487b2cc0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0487b2cc0}, {0x64a6560, 0xc04830be00}, {0xc03f653e90, 0x20339, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0487b2cc0}, {0x6466ed8, 0xc05a2da680}, {0xc03f653e90, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc044617cc8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc058269f00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c5cc00, {0x63c85b8, 0xc0487b2cc0}, 0xc058269f00, 0xc047208c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c5cc00, {0x63c85b8, 0xc0487b2cc0}, 0xc058269f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0487b2c30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0487b2c30}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0487b2c30}, {0xc00102e4e0}, 0xc058269e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0487b2bd0}, 0xc058269e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0782738c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803046 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafee9970, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc054bd3880, 0xc0435ec800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc054bd3880, {0xc0435ec800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc054bd3880, {0xc0435ec800, 0x735, 0xc03f42b540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0538faad8, {0xc0435ec800, 0x7eda79, 0xc0435ec805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04f9e1c20, {0xc0435ec800, 0x4, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc01cfdd080, {0xc0435ec800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc09486daf8, {0x631b900, 0xc01cfdd080}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc09486d880, {0x7fe42ab59310, 0xc04f9e1c20}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc09486d880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc09486d880, {0xc09db71000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01f1921e0, {0xc09db71000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc00e0d0088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc00e0d0088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc00e0d0660, {0x634aac0, 0xc00e0d0088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc00e0d0000, 0xc050a00170, 0xc095d6942b, 0xc095d6942d, {0x635bec0, 0xc053e575f0}, 0xc000d6ae00, 0xc054dc2498, 0xc095d69429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc00e0d0000, {0x63c85b8, 0xc0239c4600}, 0xc050a00160, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc095a61021, 0x8}}, 0x0, 0xc0239c4540, 0xc0239c45d0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0239c4600}, {0x643d810, 0xc04f9e1c20}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06f560510}, {0x643d810, 0xc04f9e1c20}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06f560510}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808078 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02478df20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791669 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc023b428a0}, 0xc01ad27cb0, {0x646cc48, 0xc023b42900}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc023b428a0}, 0xedb7a834e, 0xc0878178c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc023b428a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0477b66f0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01ad27ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0477b66f0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803815 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc04846ff20}, 0xc04ef3ce00, 0xc095fd3180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc04846ff20}, 0xc04ef3ce00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110306540e2, 0x0, 0x0}, 0xc052083aa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110306540e2, 0x0, 0x0}, 0xc052083aa0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04846fe90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04846fe90}, {0xc063b3d3f8, 0x660089}, 0xc063b3d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04846fe90}, {0xc0015221e0}, 0xc04ef3cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04846fe00}, 0xc04ef3cb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}, 0xc073f7e9e0, 0xc0b3d4bf80) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}, 0x3, 0xc03c4ced40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc063b3d980}}}, {0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}, 0xc073f7e9e0, 0xc03c4ced40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04846fe00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04846fe00}, {0x40, 0xc063b3db70}, 0xc063b3dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04846fe00}, {0x4e9d7a0, 0xc04ef3cb00}, 0xc073f7e9e0, 0xc0b3d4bf80) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04846fe00}, 0xc06e679500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04c2a97a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04c2a97a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804940 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02f1c6280, {0x63c85b8, 0xc03897b770}, 0xc08afc7488, 0x1, 0x708017db7933cc, 0x37e11d600, 0xc06f9cecc0, 0xc0717b84e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03897b770}, 0x120, 0xc08c351230, 0x6f9cec60, 0xc0777829c0, 0x18, 0xc06f9cecc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02f1c6280, {0x63c85b8, 0xc03897b770}, 0xc06f9cec60, {{{{0x0, 0x0}, {0xc04876d7a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03897b770}, 0x6352410, 0x52e78140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f663a40, {0x63c85b8, 0xc03897b770}, {0xc06f9ceb40, {0x174301103d365dfe, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f663a40, {0x63c85b8, 0xc03897b770}, 0xc068223900, 0xc08c3519f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03897b770}, 0x660089, {0xc06f9ceb40, {0x174301103d365dfe, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f1b9000, {0x63c85b8, 0xc03897b770}, 0xc06f10bc00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103d365dfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103d365dfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03897b590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03897b590}, {0xc08c3533f8, 0x660089}, 0xc08c353400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03897b590}, {0xc0015221e0}, 0xc06f10ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03897b4d0}, 0xc06f10ba00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}, 0xc060512680, 0xc08afc7458) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}, 0x3, 0xc016ae2e00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc08c353980}}}, {0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}, 0xc060512680, 0xc016ae2e00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03897b4d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03897b4d0}, {0x40, 0xc08c353b70}, 0xc08c353ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}, 0x203005, 0x203005) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03897b4d0}, {0x4e9d7a0, 0xc06f10ba00}, 0xc060512680, 0xc08afc7458) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03897b4d0}, 0xc064bb1680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06f9ce7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06f9ce7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802838 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01896b040, {0x63c85b8, 0xc02a290ae0}, 0xc0159bb440, 0x1, 0xc013b570a8, 0x519bbc, 0xc075efe960, 0xc06521cde0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02a290ae0}, 0x120, 0xc013b57230, 0x75efe900, 0x0, 0x660069, 0xc075efe960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01896b040, {0x63c85b8, 0xc02a290ae0}, 0xc075efe900, {{{{0x0, 0x0}, {0xc049fb5440, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02a290ae0}, 0x6352410, 0x6492bbc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01893eaa0, {0x63c85b8, 0xc02a290ae0}, {0xc075efe7e0, {0x174301101d0e96f4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01893eaa0, {0x63c85b8, 0xc02a290ae0}, 0xc0415d0960, 0xc013b579f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02a290ae0}, 0x660089, {0xc075efe7e0, {0x174301101d0e96f4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018a90400, {0x63c85b8, 0xc02a290ae0}, 0xc0a9a78b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d0e96f4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d0e96f4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02a290810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02a290810}, {0xc013b593f8, 0x660089}, 0xc013b59400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02a290810}, {0xc0615b4e10}, 0xc0a9a78900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02a290780}, 0xc0a9a78900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}, 0xc0736d9dc0, 0xc04107a210) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}, 0x3, 0xc037d0d1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc013b59980}}}, {0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}, 0xc0736d9dc0, 0xc037d0d1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02a290780}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02a290780}, {0x40, 0xc013b59b70}, 0xc013b59ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02a290780}, {0x4e9d7a0, 0xc0a9a78900}, 0xc0736d9dc0, 0xc04107a210) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02a290780}, 0xc017711080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075efe6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075efe6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809662 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa280, {0x63c85b8, 0xc07031ecc0}, 0xc022670570, 0x1, 0x0, 0x1, 0xc06f041b60, 0xc06eaeade0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07031ecc0}, 0x120, 0xc033dd2728, 0x6f041b00, 0x0, 0x660069, 0xc06f041b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa280, {0x63c85b8, 0xc07031ecc0}, 0xc06f041b00, {{{{0x0, 0x0}, {0xc0733c6360, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07031ecc0}, 0x6352410, 0x6b245108) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e7010e0, {0x63c85b8, 0xc07031ecc0}, {0xc046636900, {0x1743011065ed6dda, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e7010e0, {0x63c85b8, 0xc07031ecc0}, 0xc02e9fe960, 0xc0766b6ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07031ecc0}, 0x660089, {0xc046636900, {0x1743011065ed6dda, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edc9c00, {0x63c85b8, 0xc07031ecc0}, 0xc05f535d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ed6dda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ed6dda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07031ec30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07031ec30}, {0xc01fad68f0, 0x660089}, 0xc01fad68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07031ec30}, {0xc00102e4e0}, 0xc05f535c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07031ebd0}, 0xc05f535c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc073da4c30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790732 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe799a8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06c321780, 0xc034d8a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06c321780, {0xc034d8a000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06c321780, {0xc034d8a000, 0x735, 0xc038f3d900}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc055bf2638, {0xc034d8a000, 0x7eda79, 0xc034d8a005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc07df82120, {0xc034d8a000, 0xc0638eec60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02febe648, {0xc034d8a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc054183778, {0x631b900, 0xc02febe648}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc054183500, {0x7fe42ab59310, 0xc07df82120}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc054183500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc054183500, {0xc08fc22000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0541fec90, {0xc08fc22000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03d6c2b08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03d6c2b08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03d6c30e0, {0x634aac0, 0xc03d6c2b08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03d6c2a80, 0xc03b6266b0, 0xc095e2742b, 0xc095e2742d, {0x635bec0, 0xc09f506630}, 0xc000d6ae00, 0xc02ef4add8, 0xc095e27429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03d6c2a80, {0x63c85b8, 0xc049499aa0}, 0xc03b6266a0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc07368204d, 0x8}}, 0x0, 0xc049499980, 0xc0494999b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc049499aa0}, {0x643d810, 0xc07df82120}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc061cbb8c0}, {0x643d810, 0xc07df82120}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc061cbb8c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809528 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc08826e000}, 0xc022317188, 0x1, 0xc01af910a8, 0x519bbc, 0xc086f68600, 0xc0560e6de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08826e000}, 0x120, 0xc01af91230, 0x86f685a0, 0x0, 0x660069, 0xc086f68600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc08826e000}, 0xc086f685a0, {{{{0x0, 0x0}, {0xc05133e000, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08826e000}, 0x6352410, 0x3102ed88) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc08826e000}, {0xc086f68480, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc08826e000}, 0xc05fb45720, 0xc01af919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08826e000}, 0x660089, {0xc086f68480, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc08826e000}, 0xc05b358500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ac55da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ac55da0}, {0xc01af933f8, 0x660089}, 0xc01af93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ac55da0}, {0xc0015221e0}, 0xc05b358300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ac55bc0}, 0xc05b358300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}, 0xc024415640, 0xc034dc3908) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}, 0x3, 0xc0af72c080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01af93980}}}, {0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}, 0xc024415640, 0xc0af72c080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02ac55bc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ac55bc0}, {0x40, 0xc01af93b70}, 0xc01af93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02ac55bc0}, {0x4e9d7a0, 0xc05b358300}, 0xc024415640, 0xc034dc3908) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02ac55bc0}, 0xc06da6b860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086f68360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086f68360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802592 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc023ba7c00, {0x63c85b8, 0xc018a22a80}, 0xc04c13ea00, 0xc04ebc94a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc023ba7c00, {0x63c85b8, 0xc018a22a80}, 0xc04c13ea00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110254c99a2, 0x0, 0x0}, 0xc04827fa30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110254c99a2, 0x0, 0x0}, 0xc04827fa30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc018a229f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018a229f0}, {0xc062c413f8, 0x660089}, 0xc062c41400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc018a229f0}, {0xc04b9bbcf0}, 0xc04c13e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc018a22720}, 0xc04c13e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}, 0xc0736d8440, 0xc0571d5d58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}, 0x3, 0xc042d2d780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062c41980}}}, {0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}, 0xc0736d8440, 0xc042d2d780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc018a22720}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc018a22720}, {0x40, 0xc062c41b70}, 0xc062c41ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc018a22720}, {0x4e9d7a0, 0xc04c13e800}, 0xc0736d8440, 0xc0571d5d58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc018a22720}, 0xc077ef09c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04bf1d440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04bf1d440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812612 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc043d54380, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc041cd3198, 0xc091e77538, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc091e77650, 0x0, 0xc091e77650) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc091e774e8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091e774e8, {0xc01db54280, 0x18, 0x74}, {0xc03774da40, 0x18, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091e762c0, {0xc03774da40, 0x0, 0xc041cd34e0}, 0x121ea30, 0x20) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091e762c0, {0xc01db54280, 0x18, 0x6d}, {0xc03774da40, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091e76000, {0xc03774da40, 0x18, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0550aac08, {{0xc08467a990, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc073bd7900, {{0xc08467a990, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0bbab5800, {0x63c85b8, 0xc04f63bb90}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04f63bb90}, {0x64a6560, 0xc073bd7900}, {0xc08467a990, 0x375a2f763354ca55, 0x18}, {0x1743011095254f25, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04f63bb90}, {0x6466ed8, 0xc0550aac00}, {0xc08467a990, 0x17, 0x18}, {0x1743011095254f25, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc049c94528}, {{0x1743011095254f25, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011095254f25, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0455a3e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b2f2b425, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc04f63bb90}, 0xc0455a3e00, 0xc022382820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc04f63bb90}, 0xc0455a3e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011095254f25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011095254f25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63bb00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63bb00}, {0xc041cd93f8, 0x660089}, 0xc041cd9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63bb00}, {0x5646517a436a5373}, 0xc0455a3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63ba70}, 0xc0455a3b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}, 0xc03ebdd640, 0xc031cfb1b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}, 0x3, 0xc033d33d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc041cd9980}}}, {0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}, 0xc03ebdd640, 0xc033d33d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63ba70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63ba70}, {0x40, 0xc041cd9b70}, 0xc041cd9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63ba70}, {0x4e9d7a0, 0xc0455a3b00}, 0xc03ebdd640, 0xc031cfb1b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63ba70}, 0xc039f0acc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356a000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802139 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc0473ba600}, 0xc0654b1c00, 0xc08f98b040) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0473ba600}, 0xc0654b1c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110223eb4c6, 0x0, 0x0}, 0xc04827e710, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110223eb4c6, 0x0, 0x0}, 0xc04827e710, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0473ba570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473ba570}, {0xc05c9eb3f8, 0x660089}, 0xc05c9eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0473ba570}, {0x590000009e06}, 0xc0654b1a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0473ba480}, 0xc0654b1a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}, 0xc0871b39c0, 0xc0264bfc68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}, 0x3, 0xc05608dc40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05c9eb980}}}, {0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}, 0xc0871b39c0, 0xc05608dc40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0473ba480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0473ba480}, {0x40, 0xc05c9ebb70}, 0xc05c9ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0473ba480}, {0x4e9d7a0, 0xc0654b1a00}, 0xc0871b39c0, 0xc0264bfc68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0473ba480}, 0xc063af4480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0747fbe60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0747fbe60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803322 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc0525c9ad0}, 0xc05c7d7b00, 0xc052ea65a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0525c9ad0}, 0xc05c7d7b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102c1d0a30, 0x0, 0x0}, 0xc03ba59e10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102c1d0a30, 0x0, 0x0}, 0xc03ba59e10, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0525c9a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0525c9a40}, {0xc04dd9d3f8, 0x660089}, 0xc04dd9d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0525c9a40}, {0x7fe3f3382df8}, 0xc05c7d7900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0525c99b0}, 0xc05c7d7900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}, 0xc02982fa80, 0xc03a0b8810) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}, 0x3, 0xc0217fd780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04dd9d980}}}, {0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}, 0xc02982fa80, 0xc0217fd780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0525c99b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0525c99b0}, {0x40, 0xc04dd9db70}, 0xc04dd9dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0525c99b0}, {0x4e9d7a0, 0xc05c7d7900}, 0xc02982fa80, 0xc03a0b8810) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0525c99b0}, 0xc038f10cc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc036a2a480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc036a2a480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804761 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc016c4d000, {0x63c85b8, 0xc02c298f60}, 0xc058ad3100, 0xc02e903ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc02c298f60}, 0xc058ad3100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02c298ed0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c298ed0}, {0xc079fa93f8, 0x660089}, 0xc079fa9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02c298ed0}, {0xc0015221e0}, 0xc058ad2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02c298d50}, 0xc058ad2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}, 0xc01963ea00, 0xc03aba78d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}, 0x3, 0xc0025fe1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079fa9980}}}, {0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}, 0xc01963ea00, 0xc0025fe1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02c298d50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c298d50}, {0x40, 0xc079fa9b70}, 0xc079fa9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}, 0x203000, 0x203000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02c298d50}, {0x4e9d7a0, 0xc058ad2e00}, 0xc01963ea00, 0xc03aba78d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02c298d50}, 0xc0cc66bf20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06e73a120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06e73a120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810282 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc025d5f400, {0x63c85b8, 0xc06e4fa6c0}, 0xc0a3d8e200, 0xc052bbb5e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025d5f400, {0x63c85b8, 0xc06e4fa6c0}, 0xc0a3d8e200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c9f906b, 0x0, 0x0}, 0xc03399fa40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c9f906b, 0x0, 0x0}, 0xc03399fa40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06e4fa630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e4fa630}, {0xc01a0093f8, 0x660089}, 0xc01a009400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06e4fa630}, {0x7fe3c2f0ad80}, 0xc0a3d8e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06e4fa5a0}, 0xc0a3d8e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}, 0xc0257164a0, 0xc0cb386300) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}, 0x3, 0xc0418be680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a009980}}}, {0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}, 0xc0257164a0, 0xc0418be680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06e4fa5a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06e4fa5a0}, {0x40, 0xc01a009b70}, 0xc01a009ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06e4fa5a0}, {0x4e9d7a0, 0xc0a3d8e000}, 0xc0257164a0, 0xc0cb386300) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06e4fa5a0}, 0xc07161f500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc016409e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc016409e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812592 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800860 [semacquire]: sync.runtime_SemacquireMutex(0x676fa67, 0xc0, 0x7fd76b0e8ce0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074b2b0, 0xc000088000, 0xc095906410, 0x49e6b9) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x119405e0903924d, 0x300, 0xc, 0x9baf000) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7236e27, 0x3b81}, 0x0, 0xc0064b78f8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0064b7680, {0xffffffffffffff15, 0xc0959065b0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0064b7680) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0064b7680, {0xc02e797300, 0x32, 0x80}, 0x13, 0x15) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0064b7680, {0xc02f14ee00, 0x32, 0x80}, {0xc02e797300, 0x51c9860, 0xc02e797300}, 0x32, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0064b7680, {0xc02f14ee00, 0x32, 0x77}, {0xc02e797300, 0x32, 0x80}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07da54d98, {0xc02f14ee00, 0x32, 0x77}, {0xc02e797300, 0x32, 0x80}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07da536c0, {0xc02e797300, 0x80, 0x77}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07da536c0, {0xc02f14ee00, 0x80, 0xc02e797300}, {0xc02e797300, 0xc0703a3dc0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07da53400, {0xc02e797300, 0x32, 0x80}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0c9473328, {{0xc0703a3dc0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc058263c00, {{0xc0b8c291a0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0497f6000, {0x63c85b8, 0xc061126b70}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc061126b70}, {0x64a6560, 0xc058263c00}, {0xc0b8c291a0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc061126b70}, {0x6466ed8, 0xc0c9473080}, {0xc0b8c291a0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc046ebf050}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc063843100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c02400, {0x63c85b8, 0xc061126b70}, 0xc063843100, 0xc0b35100a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c02400, {0x63c85b8, 0xc061126b70}, 0xc063843100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc061126ae0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc061126ae0}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc061126ae0}, {0xc00102e4e0}, 0xc063843000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc061126a80}, 0xc063843000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a4840}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804479 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc027c905a0}, 0xc0c575ea00, 0xc0845bc8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc027c905a0}, 0xc0c575ea00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061978db5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061978db5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc027c90510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc027c90510}, {0xc077c547f8, 0x660089}, 0xc077c54800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc027c90510}, {0xc00102e4e0}, 0xc0c575e900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc027c904b0}, 0xc0c575e900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc01f2206e0, {0x63c85b8, 0xc027c902d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0a5f4e000, {0x63c85b8, 0xc027c902d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0a5f4e000, {0x63c8510, 0xc01bc32dc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc0a5f4e000, {0x63c8510, 0xc01bc32dc0}, {0x63f7db8, 0x9bf6f48}, 0xc077c59088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0a5f4e000, {0x63c85b8, 0xc052cd9950}, {{0x63f7db8, 0x9bf6f48}, {0xc00769607d, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0a5f4e000, {0x63c85b8, 0xc052cd9950}, {{0x63f7db8, 0x9bf6f48}, {0xc00769607d, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc00769607d, 0x6}, 0x0, 0x0}, {0xb145198, 0xedb7a8350, 0x0}, {0xb14564e, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0a5f4e000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0a5f4e000, {0x63c8510, 0xc06bd25400}, 0xc075e71dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc077396210) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06bd25400}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc077396210) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219810787 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc072424c30}, 0xc0387e4a50, 0x1, 0xc01aa4f0a8, 0x519bbc, 0xc0531bc240, 0xc020338420) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc072424c30}, 0x1e0, 0xc01aa4f230, 0x531bc1e0, 0x0, 0x660069, 0xc0531bc240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc072424c30}, 0xc0531bc1e0, {{{{0x0, 0x0}, {0xc086190240, 0x61}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc072424c30}, 0x63521a0, 0x50b34f18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc072424c30}, {0xc050215680, {0x174301106d8bb05b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc072424c30}, 0xc05953f2c0, 0xc01aa4f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc072424c30}, 0x660089, {0xc050215680, {0x174301106d8bb05b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc072424c30}, 0xc08cb13600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d8bb05b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d8bb05b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072424ba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072424ba0}, {0xc01aa513f8, 0x660089}, 0xc01aa51400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072424ba0}, {0x0}, 0xc08cb13400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072424b10}, 0xc08cb13400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}, 0xc03e2f4920, 0xc031aa1f08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}, 0x3, 0xc054254780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aa51980}}}, {0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}, 0xc03e2f4920, 0xc054254780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc072424b10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072424b10}, {0x40, 0xc01aa51b70}, 0xc01aa51ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc072424b10}, {0x4e9d7a0, 0xc08cb13400}, 0xc03e2f4920, 0xc031aa1f08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc072424b10}, 0xc044f055c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04a5b77a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04a5b77a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803288 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0647b0930}, 0xc054db1128, 0x1, 0xc0540b10a8, 0x519bbc, 0xc0447e3aa0, 0xc03fc438c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0647b0930}, 0x1e0, 0xc0540b1230, 0x447e3a40, 0x0, 0x660069, 0xc0447e3aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0647b0930}, 0xc0447e3a40, {{{{0x0, 0x0}, {0xc07e1fd440, 0x38}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0647b0930}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0647b0930}, {0xc03377fb00, {0x174301102ad31a60, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0647b0930}, 0xc050934f00, 0xc0540b19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0647b0930}, 0x660089, {0xc03377fb00, {0x174301102ad31a60, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0647b0930}, 0xc031e0a000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ad31a60, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0647b08a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0647b08a0}, {0xc0540b33f8, 0x660089}, 0xc0540b3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0647b08a0}, {0xc047221c40}, 0xc0745afd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0647b07e0}, 0xc0745afd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}, 0xc03951c920, 0xc03342a5e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}, 0x3, 0xc046788a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0540b3980}}}, {0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}, 0xc03951c920, 0xc046788a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0647b07e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0647b07e0}, {0x40, 0xc0540b3b70}, 0xc0540b3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0647b07e0}, {0x4e9d7a0, 0xc0745afd00}, 0xc03951c920, 0xc03342a5e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0647b07e0}, 0xc018726240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0345e87e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0345e87e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808496 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020c00c00, {0x63c85b8, 0xc079e709c0}, 0xc05be2ed00, 0xc062a2a140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c00c00, {0x63c85b8, 0xc079e709c0}, 0xc05be2ed00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105fdf54f9, 0x0, 0x0}, 0xc0301dd1a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105fdf54f9, 0x0, 0x0}, 0xc0301dd1a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc079e708a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc079e708a0}, {0xc076b97918, 0x660089}, 0xc076b97920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc079e708a0}, {0xc00102e4e0}, 0xc05be2e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc079e707e0}, 0xc05be2e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06f213260}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc079e70630}, {0xc00dfd36b0, 0x24}, 0x46c7b0290, 0xc028b31d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc079e70630}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219790217 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0466a6960}, 0xc077096cb0, {0x646cc48, 0xc0466a6a20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0466a6960}, 0xedb7a834e, 0xc086f6b680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0466a6960}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc047adf020}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc077096ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc047adf020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219443372 [IO wait]: internal/poll.runtime_pollWait(0x7fe33bf9ff18, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc057980f80, 0xc054f1a000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc057980f80, {0xc054f1a000, 0x2b7d, 0x2b7d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc057980f80, {0xc054f1a000, 0x2b7d, 0xc04c2823c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01b6f6558, {0xc054f1a000, 0x7eda79, 0xc054f1a005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc05f6760c0, {0xc054f1a000, 0xc03ff80c60, 0x2b7d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc048cf5b78, {0xc054f1a000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc05bb5a978, {0x631b900, 0xc048cf5b78}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc05bb5a700, {0x7fe42ab59310, 0xc05f6760c0}, 0x46a86f) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc05bb5a700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc05bb5a700, {0xc0326e0000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0140903a8, {0xc0326e0000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc09c0c8088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc09c0c8088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc09c0c8660, {0x634aac0, 0xc09c0c8088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc09c0c8000, 0xc065748fb0, 0xc03ff8142b, 0xc03ff8142d, {0x635bec0, 0xc0617c1710}, 0xc000d6ae00, 0xc03ed563d8, 0xc03ff81429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc09c0c8000, {0x63c85b8, 0xc04ba1d470}, 0xc065748fa0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc07cb71037, 0x8}}, 0x0, 0xc04ba1d410, 0xc04ba1d440, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc04ba1d470}, {0x643d810, 0xc05f6760c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0419bb530}, {0x643d810, 0xc05f6760c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0419bb530}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811288 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0210c1c00, {0x63c85b8, 0xc02677a240}, 0xc06b6e3f00, 0xc072554be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c1c00, {0x63c85b8, 0xc02677a240}, 0xc06b6e3f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110747803fb, 0x0, 0x0}, 0xc04b061e20, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110747803fb, 0x0, 0x0}, 0xc04b061e20, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02677a1b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02677a1b0}, {0xc00d79d3f8, 0x660089}, 0xc00d79d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02677a1b0}, {0xc039614eb0}, 0xc06b6e3d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02677a0c0}, 0xc06b6e3d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}, 0xc0409df200, 0xc07b35db48) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}, 0x3, 0xc047376540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00d79d980}}}, {0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}, 0xc0409df200, 0xc047376540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02677a0c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02677a0c0}, {0x40, 0xc00d79db70}, 0xc00d79dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02677a0c0}, {0x4e9d7a0, 0xc06b6e3d00}, 0xc0409df200, 0xc07b35db48) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02677a0c0}, 0xc04c155560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06e314a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06e314a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812187 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bfd92c0, {0x63c85b8, 0xc046e1b200}, 0xc044b4b8d8, 0x1, 0x7080181d7a02c2, 0x37e11d600, 0xc0140aca80, 0xc042eb8960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046e1b200}, 0x120, 0xc0383f3230, 0x140aca20, 0x0, 0x660069, 0xc0140aca80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bfd92c0, {0x63c85b8, 0xc046e1b200}, 0xc0140aca20, {{{{0x0, 0x0}, {0xc042e61b00, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046e1b200}, 0x6352410, 0x54d3a300) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bc698b0, {0x63c85b8, 0xc046e1b200}, {0xc0140ac7e0, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bc698b0, {0x63c85b8, 0xc046e1b200}, 0xc0668dafa0, 0xc0383f39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046e1b200}, 0x660089, {0xc0140ac7e0, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bffd400, {0x63c85b8, 0xc046e1b200}, 0xc0a7624200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046e1b170}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046e1b170}, {0xc0383f53f8, 0x660089}, 0xc0383f5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046e1b170}, {0x6d4c39334f372d67}, 0xc0a7624000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046e1b0e0}, 0xc0a7624000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}, 0xc06d7657a0, 0xc044b4b8a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}, 0x3, 0xc06e107b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0383f5980}}}, {0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}, 0xc06d7657a0, 0xc06e107b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046e1b0e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046e1b0e0}, {0x40, 0xc0383f5b70}, 0xc0383f5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046e1b0e0}, {0x4e9d7a0, 0xc0a7624000}, 0xc06d7657a0, 0xc044b4b8a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046e1b0e0}, 0xc04c4d9bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f31200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f31200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812277 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc04c4cc360}, 0xc03c538600, 0x1, 0xc036b7f0a8, 0x519bbc, 0xc00b705da0, 0xc09fd4c720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04c4cc360}, 0x120, 0xc036b7f230, 0xb705d40, 0x0, 0x660069, 0xc00b705da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc04c4cc360}, 0xc00b705d40, {{{{0x0, 0x0}, {0xc050092d80, 0xf}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04c4cc360}, 0x6352410, 0x33a92588) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc04c4cc360}, {0xc00b7057a0, {0x174301107e50a6b9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc04c4cc360}, 0xc02e9030e0, 0xc036b7f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c4cc360}, 0x660089, {0xc00b7057a0, {0x174301107e50a6b9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc04c4cc360}, 0xc04b09ab00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c4cc2d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c4cc2d0}, {0xc036b813f8, 0x660089}, 0xc036b81400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c4cc2d0}, {0x7fe3c2f007f0}, 0xc04b09a900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c4cc240}, 0xc04b09a900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}, 0xc0262c7c80, 0xc03c539620) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}, 0x3, 0xc07810d940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc036b81980}}}, {0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}, 0xc0262c7c80, 0xc07810d940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04c4cc240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c4cc240}, {0x40, 0xc036b81b70}, 0xc036b81ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04c4cc240}, {0x4e9d7a0, 0xc04b09a900}, 0xc0262c7c80, 0xc03c539620) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04c4cc240}, 0xc051c044e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09fd4d7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09fd4d7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800381 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc08a9d0548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc08a9d05b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0b1eef6f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc05f893180, {0xc03c3f4320, 0x2b, 0xa0}, {0xc01aed7340, 0x2b, 0x6d}, 0x1, 0xf6) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0b1eef588, {0xc03c3f4320, 0x1, 0x6d}, {0xc01aed7340, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b1eef588, {0xc03c3f4320, 0x2b, 0xa0}, {0xc01aed7340, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b1eeecc0, {0xc01aed7340, 0x6d, 0xa0}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b1eeecc0, {0xc03c3f4320, 0x2b, 0xc01aed7340}, {0xc01aed7340, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b1eeea00, {0xc01aed7340, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc044841708, {{0xc0a423af60, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc07b309800, {{0xc0a423af60, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc040431400, {0x63c85b8, 0xc045f18ae0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc045f18ae0}, {0x64a6560, 0xc07b309800}, {0xc0a423af60, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc045f18ae0}, {0x6466ed8, 0xc044841700}, {0xc0a423af60, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0293d9c80}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0975c2200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020abbc00, {0x63c85b8, 0xc045f18ae0}, 0xc0975c2200, 0xc02f38ee60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020abbc00, {0x63c85b8, 0xc045f18ae0}, 0xc0975c2200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045f18a50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045f18a50}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045f18a50}, {0xc00102e4e0}, 0xc0975c2100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045f189f0}, 0xc0975c2100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06fc7b7d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808120 [IO wait]: internal/poll.runtime_pollWait(0x7fe38e3dfd70, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc066259200, 0xc00d849000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc066259200, {0xc00d849000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc066259200, {0xc00d849000, 0x735, 0xc07a281540}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc043390230, {0xc00d849000, 0x7eda79, 0xc00d849005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06decb5c0, {0xc00d849000, 0x4dc51f3f6d990000, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc032a73170, {0xc00d849000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0be50acf8, {0x631b900, 0xc032a73170}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0be50aa80, {0x7fe42ab59310, 0xc06decb5c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0be50aa80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0be50aa80, {0xc003549000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc06582a540, {0xc003549000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04fa0c088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04fa0c088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04fa0c660, {0x634aac0, 0xc04fa0c088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04fa0c000, 0xc0540bf930, 0xc0788af42b, 0xc0788af42d, {0x635bec0, 0xc014718630}, 0xc000d6ae00, 0xc04fa00598, 0xc0788af429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04fa0c000, {0x63c85b8, 0xc0208a5860}, 0xc0540bf920, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06cd9903e, 0x8}}, 0x0, 0xc0208a5800, 0xc0208a5830, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0208a5860}, {0x643d810, 0xc06decb5c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01a97cb10}, {0x643d810, 0xc06decb5c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01a97cb10}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803857 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc071e814d0}, 0xc04cebb6e0, 0x1, 0xc07708b0a8, 0x519bbc, 0xc064c836e0, 0xc0486d7140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc071e814d0}, 0x120, 0xc07708b230, 0x64c83680, 0x0, 0x660069, 0xc064c836e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc071e814d0}, 0xc064c83680, {{{{0x0, 0x0}, {0xc0731de480, 0x1f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc071e814d0}, 0x6352410, 0x3d0ac4a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc071e814d0}, {0xc064c83560, {0x1743011015406c30, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc071e814d0}, 0xc048f10b40, 0xc07708b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc071e814d0}, 0x660089, {0xc064c83560, {0x1743011015406c30, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc071e814d0}, 0xc02a22b400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011015406c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011015406c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc071e81440}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071e81440}, {0xc07708d3f8, 0x660089}, 0xc07708d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc071e81440}, {0xc0015221e0}, 0xc02a22b200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc071e813b0}, 0xc02a22b200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}, 0xc054287a80, 0xc048fc74b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}, 0x3, 0xc06d760e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07708d980}}}, {0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}, 0xc054287a80, 0xc06d760e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc071e813b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071e813b0}, {0x40, 0xc07708db70}, 0xc07708dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc071e813b0}, {0x4e9d7a0, 0xc02a22b200}, 0xc054287a80, 0xc048fc74b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc071e813b0}, 0xc02d579aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc064c83440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc064c83440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801262 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01816a000, {0x63c85b8, 0xc066d2fe30}, 0xc07b11ef00, 0xc0413c68c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01816a000, {0x63c85b8, 0xc066d2fe30}, 0xc07b11ef00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c0ad45c, 0x0, 0x0}, 0xc06cefd590, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c0ad45c, 0x0, 0x0}, 0xc06cefd590, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066d2fda0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066d2fda0}, {0xc076115918, 0x660089}, 0xc076115920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066d2fda0}, {0xc00102e4e0}, 0xc07b11ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066d2fd40}, 0xc07b11ec00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc061d88a20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc066d2fcb0}, {0xc00dfd36b0, 0x24}, 0x46c7bed30, 0xc003501d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc06cefd370, 0xc007307478}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc066d2fcb0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803370 [IO wait]: internal/poll.runtime_pollWait(0x7fde186b2d60, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05c4d2800, 0xc046835800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05c4d2800, {0xc046835800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05c4d2800, {0xc046835800, 0x735, 0xc08def0280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc049a43730, {0xc046835800, 0x7eda79, 0xc046835805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc038c9e9c0, {0xc046835800, 0xc07ce28c60, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03f181698, {0xc046835800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc05945de78, {0x631b900, 0xc03f181698}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc05945dc00, {0x7fe42ab59310, 0xc038c9e9c0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc05945dc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc05945dc00, {0xc06bc3d000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc05897e120, {0xc06bc3d000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc046fc8088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc046fc8088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc046fc8660, {0x634aac0, 0xc046fc8088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc046fc8000, 0xc048fce410, 0xc07ce2942b, 0xc07ce2942d, {0x635bec0, 0xc0111b83f0}, 0xc000d6ae00, 0xc04f4b3ce8, 0xc07ce29429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc046fc8000, {0x63c85b8, 0xc063ccf860}, 0xc048fce400, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0688ef03b, 0x8}}, 0x0, 0xc063ccf800, 0xc063ccf830, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc063ccf860}, {0x643d810, 0xc038c9e9c0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc045803980}, {0x643d810, 0xc038c9e9c0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc045803980}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810347 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0218b1000, {0x63c85b8, 0xc07c26a750}, 0xc08c78dd00, 0xc00b5b2500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b1000, {0x63c85b8, 0xc07c26a750}, 0xc08c78dd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cd2d628, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cd2d628, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07c26a6c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c26a6c0}, {0xc027f733f8, 0x660089}, 0xc027f73400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07c26a6c0}, {0x0}, 0xc08c78db00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07c26a270}, 0xc08c78db00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}, 0xc08bdf27e0, 0xc02a74ec90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}, 0x3, 0xc00dd8b0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc027f73980}}}, {0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}, 0xc08bdf27e0, 0xc00dd8b0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07c26a270}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07c26a270}, {0x40, 0xc027f73b70}, 0xc027f73ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07c26a270}, {0x4e9d7a0, 0xc08c78db00}, 0xc08bdf27e0, 0xc02a74ec90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07c26a270}, 0xc020f3d680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00bf76fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00bf76fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219799472 [runnable]: sync.runtime_SemacquireMutex(0xc095628690, 0x8f, 0x7fe38793b500) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc095628538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc095628560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04b704000, {0xffffffffffffff15, 0xc0956286c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc04b704000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04b704000, {0xc015c7d7a0, 0x32, 0x6f}, {0xc0925f0e40, 0x32, 0x5d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0a4d48898, {0xc015c7d7a0, 0x32, 0x6f}, {0xc0925f0e40, 0x32, 0x5d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0a4d471c0, {0xc0925f0e40, 0x5d, 0x6f}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0a4d471c0, {0xc015c7d7a0, 0x5d, 0xc0925f0e40}, {0xc0925f0e40, 0xc054c5cfc0, 0x31}, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0a4d46f00, {0xc0925f0e40, 0x32, 0x5d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04f9db428, {{0xc054c5cfc0, 0x31, 0xc0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc031fa0d00, {{0xc02cc11770, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc011363000, {0x63c85b8, 0xc00f3648d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc00f3648d0}, {0x64a6560, 0xc031fa0d00}, {0xc02cc11770, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc00f3648d0}, {0x6466ed8, 0xc04f9db180}, {0xc02cc11770, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04459e7f8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc043762400, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209f1c00, {0x63c85b8, 0xc00f3648d0}, 0xc043762400, 0xc0388f1360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209f1c00, {0x63c85b8, 0xc00f3648d0}, 0xc043762400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc00f364840}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00f364840}, {0xc0933e48f0, 0x660089}, 0xc0933e48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc00f364840}, {0xc00102e4e0}, 0xc043762200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc00f3647e0}, 0xc043762200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d020}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812596 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812035 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc028798600}, 0xc0433bb4a0, 0x1, 0x0, 0x1, 0xc085b9fc80, 0xc073f1bb60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc028798600}, 0x120, 0xc03560c728, 0x85b9fc20, 0x0, 0x660069, 0xc085b9fc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc028798600}, 0xc085b9fc20, {{{{0x0, 0x0}, {0xc0457ba6c0, 0x33}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc028798600}, 0x6352410, 0x204d2390) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc028798600}, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc028798600}, 0xc0bcee0000, 0xc07c880ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc028798600}, 0x660089, {0xc00f88a900, {0x174301106d3c0ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc028798600}, 0xc060fbb700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d3c0ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc028798570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc028798570}, {0xc0ad8368f0, 0x660089}, 0xc0ad8368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc028798570}, {0xc00102e4e0}, 0xc060fbb600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0287984e0}, 0xc060fbb600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01f1b75f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810227 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc017d58060}, 0xc035a04120, 0x1, 0xc0113bdff0, 0x519bbc, 0xc07961b6e0, 0xc0159b7500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc017d58060}, 0x1e0, 0xc0113be178, 0x7961b680, 0x0, 0x660069, 0xc07961b6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc017d58060}, 0xc07961b680, {{{{0x0, 0x0}, {0xc084427680, 0x13}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc017d58060}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc017d58060}, {0xc05a72aea0, {0x1743011012117d79, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc017d58060}, 0xc048083cc0, 0xc07cf22938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc017d58060}, 0x660089, {0xc05a72aea0, {0x1743011012117d79, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc017d58060}, 0xc0551b3b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011012117d79, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011012117d79, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056359fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056359fb0}, {0xc07cf24340, 0x660089}, 0xc07cf24348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056359fb0}, {0xc00102e4e0}, 0xc0551b3a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056359f50}, 0xc0551b3a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc07412bb00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811607 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc06b914030}, 0xc000f12108, 0x1, 0xc010d990a8, 0x519bbc, 0xc0517b3980, 0xc072108e40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc020816540, {0x63c85b8, 0xc06b914030}, 0xc059767560, 0xc010d991a8, 0x517b3980, 0x0, 0x2, 0xc0517b3980, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc06b914030}, 0xc0517b38c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04e12d200, 0xae}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06b914030}, 0x6352260, 0x2c65fad0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc06b914030}, {0xc0517b37a0, {0x174301103f3ec9da, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc06b914030}, 0xc068611b80, 0xc010d999f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b914030}, 0x660089, {0xc0517b37a0, {0x174301103f3ec9da, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc06b914030}, 0xc0574add00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f3ec9da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f3ec9da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06d29bf50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d29bf50}, {0xc010d9b3f8, 0x660089}, 0xc010d9b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06d29bf50}, {0x7fe423fe92f0}, 0xc0574adb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06d29bec0}, 0xc0574adb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}, 0xc044a36800, 0xc000f13008) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}, 0x3, 0xc0632012c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010d9b980}}}, {0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}, 0xc044a36800, 0xc0632012c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06d29bec0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d29bec0}, {0x40, 0xc010d9bb70}, 0xc010d9bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06d29bec0}, {0x4e9d7a0, 0xc0574adb00}, 0xc044a36800, 0xc000f13008) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06d29bec0}, 0xc02abb6180, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0517b3680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0517b3680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805364 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016c3b2c0, {0x63c85b8, 0xc0515a9bc0}, 0xc047826270, 0x1, 0x708017ddfc995a, 0x37e11d600, 0xc05a49d4a0, 0xc059e65b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0515a9bc0}, 0x120, 0xc05f855230, 0x5a49d440, 0xc0471d7380, 0x18, 0xc05a49d4a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016c3b2c0, {0x63c85b8, 0xc0515a9bc0}, 0xc05a49d440, {{{{0x0, 0x0}, {0xc060c65320, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0515a9bc0}, 0x6352410, 0x3d0ad9a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce21e0, {0x63c85b8, 0xc0515a9bc0}, {0xc05a49d320, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce21e0, {0x63c85b8, 0xc0515a9bc0}, 0xc058580dc0, 0xc05f8559f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0515a9bc0}, 0x660089, {0xc05a49d320, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016d2a000, {0x63c85b8, 0xc0515a9bc0}, 0xc057885000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0515a9b30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0515a9b30}, {0xc05f8573f8, 0x660089}, 0xc05f857400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0515a9b30}, {0x7fe42b75cfd8}, 0xc057884e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0515a9aa0}, 0xc057884e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}, 0xc04957a520, 0xc047826240) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}, 0x3, 0xc07d113fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05f857980}}}, {0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}, 0xc04957a520, 0xc07d113fc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0515a9aa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0515a9aa0}, {0x40, 0xc05f857b70}, 0xc05f857ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}, 0x20301f, 0x20301f) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0515a9aa0}, {0x4e9d7a0, 0xc057884e00}, 0xc04957a520, 0xc047826240) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0515a9aa0}, 0xc02ea4e660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a49cfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a49cfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804969 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff69e00, {0x63c85b8, 0xc01b30c600}, 0xc05fe663a8, 0x1, 0x708017d86e29c5, 0x37e11d600, 0xc06eaf3920, 0xc04edecf00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01b30c600}, 0x120, 0xc05b499230, 0x6eaf38c0, 0xc0471d7520, 0x18, 0xc06eaf3920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff69e00, {0x63c85b8, 0xc01b30c600}, 0xc06eaf38c0, {{{{0x0, 0x0}, {0xc05907cd80, 0x5}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01b30c600}, 0x6352410, 0x421b18c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030006be0, {0x63c85b8, 0xc01b30c600}, {0xc06eaf37a0, {0x1743011030e52d02, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030006be0, {0x63c85b8, 0xc01b30c600}, 0xc01c753720, 0xc05b4999f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01b30c600}, 0x660089, {0xc06eaf37a0, {0x1743011030e52d02, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030040800, {0x63c85b8, 0xc01b30c600}, 0xc01aa11d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011030e52d02, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011030e52d02, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01b30c570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b30c570}, {0xc05b49b3f8, 0x660089}, 0xc05b49b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01b30c570}, {0x7fe3f1e6c558}, 0xc01aa11b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01b30c4e0}, 0xc01aa11b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}, 0xc013bebee0, 0xc05fe66378) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}, 0x3, 0xc06f13dec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05b49b980}}}, {0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}, 0xc013bebee0, 0xc06f13dec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01b30c4e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01b30c4e0}, {0x40, 0xc05b49bb70}, 0xc05b49bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01b30c4e0}, {0x4e9d7a0, 0xc01aa11b00}, 0xc013bebee0, 0xc05fe66378) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01b30c4e0}, 0xc02a4cd020, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06eaf3320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06eaf3320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800131 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0871c4548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0871c45b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0b23ebbf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0761b0a80, {0xc02bf79680, 0x2b, 0x74}, {0xc0227fb500, 0x2b, 0x6b}, 0x1, 0xbb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0b23eba88, {0xc02bf79680, 0x1, 0x6b}, {0xc0227fb500, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b23eba88, {0xc02bf79680, 0x2b, 0x74}, {0xc0227fb500, 0x2b, 0x6b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b23eb1c0, {0xc0227fb500, 0x6b, 0x74}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b23eb1c0, {0xc02bf79680, 0x2b, 0xc0227fb500}, {0xc0227fb500, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b23eaf00, {0xc0227fb500, 0x2b, 0x6b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc03e300588, {{0xc06568eba0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04256ca00, {{0xc06568eba0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc029e33400, {0x63c85b8, 0xc047e5b080}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc047e5b080}, {0x64a6560, 0xc04256ca00}, {0xc06568eba0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc047e5b080}, {0x6466ed8, 0xc03e300580}, {0xc06568eba0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02329bfc8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc073cb6800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0209f2800, {0x63c85b8, 0xc047e5b080}, 0xc073cb6800, 0xc073bbbb80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0209f2800, {0x63c85b8, 0xc047e5b080}, 0xc073cb6800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc047e5aff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc047e5aff0}, {0xc0abc508f0, 0x660089}, 0xc0abc508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc047e5aff0}, {0xc00102e4e0}, 0xc073cb6700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc047e5af30}, 0xc073cb6700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d170}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810280 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc024e4a000, {0x63c85b8, 0xc04ff0c630}, 0xc059955f00, 0xc01858b180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024e4a000, {0x63c85b8, 0xc04ff0c630}, 0xc059955f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106ca13f6f, 0x0, 0x0}, 0xc03cc76cc0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106ca13f6f, 0x0, 0x0}, 0xc03cc76cc0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ff0c5a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ff0c5a0}, {0xc01a0093f8, 0x660089}, 0xc01a009400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ff0c5a0}, {0x7fe3c2f0ad80}, 0xc059955d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ff0c360}, 0xc059955d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}, 0xc00f3f5320, 0xc03135b320) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}, 0x3, 0xc057070b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a009980}}}, {0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}, 0xc00f3f5320, 0xc057070b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ff0c360}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ff0c360}, {0x40, 0xc01a009b70}, 0xc01a009ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ff0c360}, {0x4e9d7a0, 0xc059955d00}, 0xc00f3f5320, 0xc03135b320) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ff0c360}, 0xc0bb53f1a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc016409200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc016409200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809988 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02bcd0a00, {0x63c85b8, 0xc022bfc510}, 0xc02f3ddde8, 0x1, 0x70801805d99514, 0x37e11d600, 0xc073f77260, 0xc071f57b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc022bfc510}, 0x120, 0xc028239230, 0x73f77200, 0x0, 0x660069, 0xc073f77260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02bcd0a00, {0x63c85b8, 0xc022bfc510}, 0xc073f77200, {{{{0x0, 0x0}, {0xc06118d9e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc022bfc510}, 0x6352410, 0x4b1bd218) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bcc4730, {0x63c85b8, 0xc022bfc510}, {0xc073f770e0, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bcc4730, {0x63c85b8, 0xc022bfc510}, 0xc07645f400, 0xc0282399f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022bfc510}, 0x660089, {0xc073f770e0, {0x17430110533d777c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bcfa000, {0x63c85b8, 0xc022bfc510}, 0xc02927d800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110533d777c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022bfc480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022bfc480}, {0xc02823b3f8, 0x660089}, 0xc02823b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022bfc480}, {0x7fe3c14e62e8}, 0xc02927d500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022bfc3f0}, 0xc02927d500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}, 0xc03bcc63a0, 0xc02f3dddb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}, 0x3, 0xc03f66c2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02823b980}}}, {0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}, 0xc03bcc63a0, 0xc03f66c2c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022bfc3f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022bfc3f0}, {0x40, 0xc02823bb70}, 0xc02823bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022bfc3f0}, {0x4e9d7a0, 0xc02927d500}, 0xc03bcc63a0, 0xc02f3dddb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022bfc3f0}, 0xc0a3017e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c5b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0679c5b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805767 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016f5d7c0, {0x63c85b8, 0xc053e4df50}, 0xc0a2f8e6a8, 0x1, 0xc005ae50a8, 0x519bbc, 0xc03242e3c0, 0xc058f70840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc053e4df50}, 0x120, 0xc005ae5230, 0x3242e360, 0x0, 0x660069, 0xc03242e3c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016f5d7c0, {0x63c85b8, 0xc053e4df50}, 0xc03242e360, {{{{0x0, 0x0}, {0xc012e225a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc053e4df50}, 0x6352410, 0x4040c9a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016fda3c0, {0x63c85b8, 0xc053e4df50}, {0xc03242e240, {0x174301103f87867f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016fda3c0, {0x63c85b8, 0xc053e4df50}, 0xc0bcee03c0, 0xc005ae59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc053e4df50}, 0x660089, {0xc03242e240, {0x174301103f87867f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016fe8800, {0x63c85b8, 0xc053e4df50}, 0xc072039e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053e4dec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053e4dec0}, {0xc005ae73f8, 0x660089}, 0xc005ae7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053e4dec0}, {0xc0015221e0}, 0xc072039c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053e4de30}, 0xc072039c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}, 0xc029f22340, 0xc07df1f8f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}, 0x3, 0xc066321c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ae7980}}}, {0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}, 0xc029f22340, 0xc066321c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc053e4de30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053e4de30}, {0x40, 0xc005ae7b70}, 0xc005ae7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc053e4de30}, {0x4e9d7a0, 0xc072039c00}, 0xc029f22340, 0xc07df1f8f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc053e4de30}, 0xc0929dcf60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03242e120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03242e120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219787706 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafee9d30, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc068669a00, 0xc085094800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc068669a00, {0xc085094800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc068669a00, {0xc085094800, 0x735, 0xc04599c280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01fae7908, {0xc085094800, 0x7eda79, 0xc085094805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc08db86ba0, {0xc085094800, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04419f788, {0xc085094800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0b917d078, {0x631b900, 0xc04419f788}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0b917ce00, {0x7fe42ab59310, 0xc08db86ba0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0b917ce00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0b917ce00, {0xc09e5fa000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc09c25ad20, {0xc09e5fa000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0aaddeb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0aaddeb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0aaddf0e0, {0x634aac0, 0xc0aaddeb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0aaddea80, 0xc04553d0e0, 0xc09c3e942b, 0xc09c3e942d, {0x635bec0, 0xc046ee07e0}, 0xc000d6ae00, 0xc04a75e018, 0xc09c3e9429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0aaddea80, {0x63c85b8, 0xc0769d96e0}, 0xc04553d0d0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc08fa3903b, 0x8}}, 0x0, 0xc0769d9680, 0xc0769d96b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0769d96e0}, {0x643d810, 0xc08db86ba0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc078f17c80}, {0x643d810, 0xc08db86ba0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc078f17c80}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806774 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc08e1984b0}, 0xc06ea01b78, 0x1, 0xc002c0d0a8, 0x519bbc, 0xc05f115320, 0xc05f84e240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc08e1984b0}, 0x1e0, 0xc002c0d230, 0x5f1152c0, 0x0, 0x660069, 0xc05f115320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc08e1984b0}, 0xc05f1152c0, {{{{0x0, 0x0}, {0xc093ef9680, 0x111}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc02784f680, {0x63c85b8, 0xc08e1984b0}, 0xc07d45c610, 0x2c0d428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc08e1984b0}, {0xc066720a20, {0x174301105029c034, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc08e1984b0}, 0xc0702ccaa0, 0xc002c0d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08e1984b0}, 0x660089, {0xc066720a20, {0x174301105029c034, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc08e1984b0}, 0xc06b139200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105029c034, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105029c034, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08e198420}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08e198420}, {0xc002c0f3f8, 0x660089}, 0xc002c0f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08e198420}, {0x0}, 0xc06b139000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08e198390}, 0xc06b139000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}, 0xc08d491340, 0xc03c2e94b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}, 0x3, 0xc08cc3edc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002c0f980}}}, {0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}, 0xc08d491340, 0xc08cc3edc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08e198390}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08e198390}, {0x40, 0xc002c0fb70}, 0xc002c0fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08e198390}, {0x4e9d7a0, 0xc06b139000}, 0xc08d491340, 0xc03c2e94b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08e198390}, 0xc07a7790e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc066720900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc066720900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808342 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc071c3a9f0}, 0xc0169b1ef0, 0x1, 0x0, 0x1, 0xc04ec64ba0, 0xc06e4b3140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc071c3a9f0}, 0x1e0, 0xc005c50728, 0x4ec64b40, 0x0, 0x660069, 0xc04ec64ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc071c3a9f0}, 0xc04ec64b40, {{{{0x0, 0x0}, {0xc0748c4fc0, 0x52}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc071c3a9f0}, 0x6352410, 0x2ad6f508) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc071c3a9f0}, {0xc06b98aa20, {0x17430110567df5e4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc071c3a9f0}, 0xc05f31ca00, 0xc07c63eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc071c3a9f0}, 0x660089, {0xc06b98aa20, {0x17430110567df5e4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc071c3a9f0}, 0xc057137600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110567df5e4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110567df5e4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc071c3a960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc071c3a960}, {0xc0025148f0, 0x660089}, 0xc0025148f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc071c3a960}, {0xc00102e4e0}, 0xc057137500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc071c3a900}, 0xc057137500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01ff75080}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805253 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc02f9b81e0}, 0xc08afc7770, 0x1, 0xc0777db0a8, 0x519bbc, 0xc055b216e0, 0xc0587cc060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02f9b81e0}, 0x1e0, 0xc0777db230, 0x55b21680, 0xc097f52820, 0x18, 0xc055b216e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc02f9b81e0}, 0xc055b21680, {{{{0x0, 0x0}, {0xc058da0000, 0x3d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc07c41d9e0, {0x63c85b8, 0xc02f9b81e0}, 0xc00de41950, 0x777db428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc02f9b81e0}, {0xc059404000, {0x174301103f132ab9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc02f9b81e0}, 0xc0840e26e0, 0xc0777db9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02f9b81e0}, 0x660089, {0xc059404000, {0x174301103f132ab9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc02f9b81e0}, 0xc07afe1800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f132ab9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f132ab9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f9b8060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f9b8060}, {0xc0777dd3f8, 0x660089}, 0xc0777dd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f9b8060}, {0xc0015221e0}, 0xc07afe1600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc015e3bd70}, 0xc07afe1600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}, 0xc060513720, 0xc0395ce528) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}, 0x3, 0xc08a022140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0777dd980}}}, {0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}, 0xc060513720, 0xc08a022140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc015e3bd70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015e3bd70}, {0x40, 0xc0777ddb70}, 0xc0777ddab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}, 0x203022, 0x203022) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc015e3bd70}, {0x4e9d7a0, 0xc07afe1600}, 0xc060513720, 0xc0395ce528) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc015e3bd70}, 0xc0386793e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057e7fe60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc057e7fe60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802477 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b078c00, {0x63c85b8, 0xc011292960}, 0xc03285e700, 0xc04ac6bb80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b078c00, {0x63c85b8, 0xc011292960}, 0xc03285e700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102555e81b, 0x0, 0x0}, 0xc03ba34890, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102555e81b, 0x0, 0x0}, 0xc03ba34890, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc011292630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc011292630}, {0xc062c413f8, 0x660089}, 0xc062c41400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc011292630}, {0x7fe338117418}, 0xc059201f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0247b3fb0}, 0xc059201f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}, 0xc068275bc0, 0xc01d516090) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}, 0x3, 0xc0d43efc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc062c41980}}}, {0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}, 0xc068275bc0, 0xc0d43efc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0247b3fb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0247b3fb0}, {0x40, 0xc062c41b70}, 0xc062c41ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0247b3fb0}, {0x4e9d7a0, 0xc059201f00}, 0xc068275bc0, 0xc01d516090) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0247b3fb0}, 0xc0543f4fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04732bc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04732bc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810204 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc073792600}, 0xc030342a68, 0x1, 0xc0234a90a8, 0x519bbc, 0xc05c36bc80, 0xc05a761500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc073792600}, 0x1e0, 0xc0234a9230, 0x5c36bc20, 0x0, 0x660069, 0xc05c36bc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc073792600}, 0xc05c36bc20, {{{{0x0, 0x0}, {0xc0b3310b40, 0x39}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc043856360, {0x63c85b8, 0xc073792600}, 0xc05f4e77f0, 0x234a9428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc073792600}, {0xc073b33d40, {0x174301106700075e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc073792600}, 0xc05ad72f00, 0xc0234a99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073792600}, 0x660089, {0xc073b33d40, {0x174301106700075e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc073792600}, 0xc089187600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106700075e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106700075e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073792570}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073792570}, {0xc0234ab3f8, 0x660089}, 0xc0234ab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073792570}, {0xc0015221e0}, 0xc089187400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0737924e0}, 0xc089187400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}, 0xc01ddddac0, 0xc036134138) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}, 0x3, 0xc08a21cd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0234ab980}}}, {0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}, 0xc01ddddac0, 0xc08a21cd80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0737924e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0737924e0}, {0x40, 0xc0234abb70}, 0xc0234abab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0737924e0}, {0x4e9d7a0, 0xc089187400}, 0xc01ddddac0, 0xc036134138) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0737924e0}, 0xc0b4895500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc073b33c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc073b33c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812325 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc02bfd4d20}, 0xc026a0aa20, 0x1, 0x0, 0x1, 0xc027eb6600, 0xc0401f6600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02bfd4d20}, 0x1e0, 0xc039182728, 0x27eb65a0, 0x0, 0x660069, 0xc027eb6600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc02bfd4d20}, 0xc027eb65a0, {{{{0x0, 0x0}, {0xc0af5befc0, 0x6b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc0262c7dc0, {0x63c85b8, 0xc02bfd4d20}, 0xc06c6de780, 0x6da6b020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc02bfd4d20}, {0xc05153db00, {0x174301107e37978a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc02bfd4d20}, 0xc083e93040, 0xc07cdf4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02bfd4d20}, 0x660089, {0xc05153db00, {0x174301107e37978a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc02bfd4d20}, 0xc07a4c2400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e37978a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e37978a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02bfd4c00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02bfd4c00}, {0xc0562c48f0, 0x660089}, 0xc0562c48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02bfd4c00}, {0xc00102e4e0}, 0xc07a4c2300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02bfd4ba0}, 0xc07a4c2300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02bfd4000}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809350 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc05375b110}, 0xc0002e7f80, 0x1, 0xc00e369ff0, 0x519bbc, 0xc047dddaa0, 0xc039514f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc05375b110}, 0x1e0, 0xc00e36a178, 0x47ddda40, 0x0, 0x660069, 0xc047dddaa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc05375b110}, 0xc047ddda40, {{{{0x0, 0x0}, {0xc04d19e6c0, 0xac}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc05375b110}, 0x0, 0xe36a398) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc05375b110}, {0xc0aef8dc20, {0x1743011063778c91, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc05375b110}, 0xc08e181180, 0xc00b2c0938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05375b110}, 0x660089, {0xc0aef8dc20, {0x1743011063778c91, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc05375b110}, 0xc04b483700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063778c91, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063778c91, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05375b080}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05375b080}, {0xc00b2c2340, 0x660089}, 0xc00b2c2348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05375b080}, {0xc00102e4e0}, 0xc04b483600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05375b020}, 0xc04b483600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0234c2600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805739 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc023b38e40}, 0xc03dc64b70, 0x1, 0xc00503f0a8, 0x519bbc, 0xc0910f1e00, 0xc0662afaa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0430e4fc0, {0x63c85b8, 0xc023b38e40}, 0xc0486f8000, 0xc00503f1a8, 0x910f1e00, 0x0, 0x2, 0xc0910f1e00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc023b38e40}, 0xc0910f1d40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc093714fc0, 0x76}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc023b38e40}, 0x6352260, 0x6618a568) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc023b38e40}, {0xc0910f1c20, {0x174301100c6d2837, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc023b38e40}, 0xc072b90a00, 0xc00503f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc023b38e40}, 0x660089, {0xc0910f1c20, {0x174301100c6d2837, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc023b38e40}, 0xc0653e4a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100c6d2837, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100c6d2837, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023b38d80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023b38d80}, {0xc0050413f8, 0x660089}, 0xc005041400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023b38d80}, {0x0}, 0xc0653e4800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc023b38ba0}, 0xc0653e4800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}, 0xc073550380, 0xc03dc65230) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}, 0x3, 0xc08214f000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005041980}}}, {0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}, 0xc073550380, 0xc08214f000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc023b38ba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023b38ba0}, {0x40, 0xc005041b70}, 0xc005041ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc023b38ba0}, {0x4e9d7a0, 0xc0653e4800}, 0xc073550380, 0xc03dc65230) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc023b38ba0}, 0xc06e632d80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0910f1b00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0910f1b00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812565 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812255 [runnable]: github.com/cockroachdb/pebble/internal/manifest.(*LevelIterator).empty(0xc09316aaa0) github.com/cockroachdb/pebble/internal/manifest/external/com_github_cockroachdb_pebble/internal/manifest/level_metadata.go:332 +0x126 github.com/cockroachdb/pebble/internal/manifest.(*LevelIterator).SeekGE(0x4e, 0x51c9860, {0xc0505b8a80, 0x48, 0x6d}) github.com/cockroachdb/pebble/internal/manifest/external/com_github_cockroachdb_pebble/internal/manifest/level_metadata.go:405 +0x36 github.com/cockroachdb/pebble.(*levelIter).findFileGE(0xc09316a938, {0xc0505b8a80, 0x48, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:235 +0x50 github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09316a938, {0xc06d8f78c0, 0x48, 0x52}, {0xc0505b8a80, 0x48, 0x6d}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xad github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc093169bc0, {0xc0505b8a80, 0x0, 0xc043cb34e0}, 0x121ea30, 0x10) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc093169bc0, {0xc06d8f78c0, 0x48, 0x6d}, {0xc0505b8a80, 0x47, 0x50}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093169900, {0xc0505b8a80, 0x48, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0461f5b88, {{0xc03fd146e0, 0x47, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc017346000, {{0xc03fd146e0, 0x47, 0x50}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05f462800, {0x63c85b8, 0xc02f5e0780}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02f5e0780}, {0x64a6560, 0xc017346000}, {0xc03fd146e0, 0x0, 0x50}, {0x174301109eebd484, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02f5e0780}, {0x6466ed8, 0xc0461f5b80}, {0xc03fd146e0, 0x47, 0x50}, {0x174301109eebd484, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc05f255548}, {{0x174301109eebd484, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301109eebd484, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc077e4ce00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110bcb93984, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc000e4e800, {0x63c85b8, 0xc02f5e0780}, 0xc077e4ce00, 0xc022ff0460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000e4e800, {0x63c85b8, 0xc02f5e0780}, 0xc077e4ce00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301109eebd484, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301109eebd484, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f5e0660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f5e0660}, {0xc043cb93f8, 0x660089}, 0xc043cb9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f5e0660}, {0x0}, 0xc077e4cc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f5e0360}, 0xc077e4cc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}, 0xc0a169ac80, 0xc0257a2720) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}, 0x3, 0xc00e947b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043cb9980}}}, {0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}, 0xc0a169ac80, 0xc00e947b40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f5e0360}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f5e0360}, {0x40, 0xc043cb9b70}, 0xc043cb9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f5e0360}, {0x4e9d7a0, 0xc077e4cc00}, 0xc0a169ac80, 0xc0257a2720) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f5e0360}, 0xc06612c720, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc053ef3440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc053ef3440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809148 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc0ccfccf30}, 0xc05f2555f0, 0x1, 0xc014a410a8, 0x519bbc, 0xc05f77e180, 0xc051a61ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0ccfccf30}, 0x120, 0xc014a41230, 0x5f77e120, 0x0, 0x660069, 0xc05f77e180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc0ccfccf30}, 0xc05f77e120, {{{{0x0, 0x0}, {0xc089392900, 0x30}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0ccfccf30}, 0x6352410, 0x6bab3628) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc0ccfccf30}, {0xc05f77e000, {0x174301106346bdcd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc0ccfccf30}, 0xc078b59ea0, 0xc014a419f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0ccfccf30}, 0x660089, {0xc05f77e000, {0x174301106346bdcd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc0ccfccf30}, 0xc077e16e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106346bdcd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106346bdcd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0ccfccea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ccfccea0}, {0xc014a433f8, 0x660089}, 0xc014a43400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0ccfccea0}, {0xc096362230}, 0xc077e16b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0ccfcce10}, 0xc077e16b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}, 0xc03e8c68a0, 0xc013fc1a58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}, 0x3, 0xc054ffd600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc014a43980}}}, {0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}, 0xc03e8c68a0, 0xc054ffd600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0ccfcce10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ccfcce10}, {0x40, 0xc014a43b70}, 0xc014a43ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0ccfcce10}, {0x4e9d7a0, 0xc077e16b00}, 0xc03e8c68a0, 0xc013fc1a58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0ccfcce10}, 0xc04e1b3b00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07adf8ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07adf8ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800976 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc058442de0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802176 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc04c070900}, {0xc060ebedc0, 0x4, 0xc001b0c658}, {0x0, {0x1743010fef31f8a2, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc04c070900}, 0x848a, 0xc09ca29680, 0x20, 0xc032923690) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc04c070900}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802624 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc04f1fbb00}, {0xc0728ac700, 0x1, 0xc0660da198}, {0x1, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntent(0x0, {0x63c85b8, 0xc04f1fbb00}, {{{0xc0c599ec90, 0x22, 0x30}, {0x0, 0x0, 0x0}}, {{0x4e, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:841 +0x9b github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc02fea22d0, {0x63c85b8, 0xc04f1fbb00}, {0xc047372900, {0x174301101f6ccd0d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:549 +0x6db github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc04f1fbb00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc02fea22d0, {0x63c85b8, 0xc04f1fbb00}, {0xc047372900, {0x174301101f6ccd0d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc04f1fbb00}, 0xc02225ee60, 0xc0728ad558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f1fbb00}, 0xc02225ee60, {0xc047372900, {0x174301101f6ccd0d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc04f1fbb00}, 0xc0b5831600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101f6ccd0d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101f6ccd0d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07301ffb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07301ffb0}, {0xc0501b93f8, 0x660089}, 0xc0501b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07301ffb0}, {0xc0015221e0}, 0xc0b5831400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07301ff20}, 0xc0b5831400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}, 0xc0829bf940, 0xc02010fe30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}, 0x3, 0xc054b07ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0501b9980}}}, {0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}, 0xc0829bf940, 0xc054b07ec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07301ff20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07301ff20}, {0x40, 0xc0501b9b70}, 0xc0501b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07301ff20}, {0x4e9d7a0, 0xc0b5831400}, 0xc0829bf940, 0xc02010fe30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07301ff20}, 0xc073864c60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c488ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05c488ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802679 [runnable]: sync.runtime_SemacquireMutex(0xc073a38690, 0x8f, 0x7fe289a98740) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc073a38538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x38, 0x2b, 0x38, 0xc073a38560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0086ac000, {0xffffffffffffff15, 0xc073a386c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0086ac000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0086ac000, {0xc0141bae00, 0x32, 0x32}, {0xc0141badc0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091ead298, {0xc0141bae00, 0x32, 0x32}, {0xc0141badc0, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091eabbc0, {0xc0141badc0, 0x32, 0x32}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091eabbc0, {0xc0141bae00, 0x32, 0xc0141badc0}, {0xc0141badc0, 0xc051dd0b60, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091eab900, {0xc0141badc0, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc087acc828, {{0xc051dd0b60, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0840bbd00, {{0xc085967b60, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0b3d4f000, {0x63c85b8, 0xc063221020}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc063221020}, {0x64a6560, 0xc0840bbd00}, {0xc085967b60, 0x21036, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc063221020}, {0x6466ed8, 0xc087acc580}, {0xc085967b60, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc068a74dc8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc019ed7300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4f400, {0x63c85b8, 0xc063221020}, 0xc019ed7300, 0xc041b80c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4f400, {0x63c85b8, 0xc063221020}, 0xc019ed7300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc063220f90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc063220f90}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc063220f90}, {0xc00102e4e0}, 0xc019ed7200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc063220f30}, 0xc019ed7200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220ea0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808239 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6640, {0x63c85b8, 0xc046f347e0}, 0xc0443392a8, 0x1, 0xc005ab50a8, 0x519bbc, 0xc05c351a40, 0xc0b9b52760) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046f347e0}, 0x120, 0xc005ab5230, 0x5c3519e0, 0x0, 0x660069, 0xc05c351a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6640, {0x63c85b8, 0xc046f347e0}, 0xc05c3519e0, {{{{0x0, 0x0}, {0xc06540e6c0, 0x10}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046f347e0}, 0x63526b0, 0x2eda5168) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdde50, {0x63c85b8, 0xc046f347e0}, {0xc05c3518c0, {0x174301105dbbb8e7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdde50, {0x63c85b8, 0xc046f347e0}, 0xc0549c1ea0, 0xc005ab59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046f347e0}, 0x660089, {0xc05c3518c0, {0x174301105dbbb8e7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0b2c00, {0x63c85b8, 0xc046f347e0}, 0xc039e4f400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105dbbb8e7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105dbbb8e7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046f34750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046f34750}, {0xc005ab73f8, 0x660089}, 0xc005ab7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046f34750}, {0x7fdd4a3744c0}, 0xc039e4f200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046f346c0}, 0xc039e4f200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}, 0xc0748d5ca0, 0xc0198abda0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}, 0x3, 0xc04e354540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005ab7980}}}, {0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}, 0xc0748d5ca0, 0xc04e354540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046f346c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046f346c0}, {0x40, 0xc005ab7b70}, 0xc005ab7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046f346c0}, {0x4e9d7a0, 0xc039e4f200}, 0xc0748d5ca0, 0xc0198abda0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046f346c0}, 0xc04acaea80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05c351680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05c351680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802677 [runnable]: sync.runtime_SemacquireMutex(0xc05105c690, 0x8f, 0x7fdbb3bc3bc0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc05105c538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc05105c560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc03e1f6480, {0xffffffffffffff15, 0xc05105c6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc03e1f6480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc03e1f6480, {0xc00795da40, 0x32, 0x32}, {0xc00795d140, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09283c398, {0xc00795da40, 0x32, 0x32}, {0xc00795d140, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09283acc0, {0xc00795d140, 0x32, 0x32}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09283acc0, {0xc00795da40, 0x32, 0xc00795d140}, {0xc00795d140, 0xc06197d6c0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09283aa00, {0xc00795d140, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc091182ea8, {{0xc06197d6c0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0a0753400, {{0xc03e50e660, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07cbf6800, {0x63c85b8, 0xc050e352f0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc050e352f0}, {0x64a6560, 0xc0a0753400}, {0xc03e50e660, 0x31, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc050e352f0}, {0x6466ed8, 0xc091182c00}, {0xc03e50e660, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0240afba8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0890a8800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4dc00, {0x63c85b8, 0xc050e352f0}, 0xc0890a8800, 0xc017408dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4dc00, {0x63c85b8, 0xc050e352f0}, 0xc0890a8800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc050e35230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc050e35230}, {0xc0076cc8f0, 0x660089}, 0xc0076cc8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc050e35230}, {0xc00102e4e0}, 0xc0890a8700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc050e35110}, 0xc0890a8700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220db0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219607201 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8510, 0xc079b79880}, 0x0, 0xc0b0fd6580) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc0385a9d90, {0x63c8510, 0xc079b79880}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/txn.go:696 +0x74 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc090091b30, {0x63c8510, 0xc079b79880}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:225 +0xb6 github.com/cockroachdb/cockroach/pkg/sql.(*insertFastPathNode).BatchedNext(0xc090091b00, {{0x63c8510, 0xc079b79880}, 0xc026ebf180, 0xc054d239f0}) github.com/cockroachdb/cockroach/pkg/sql/insert_fast_path.go:311 +0x127 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc03879f620, {{0x63c8510, 0xc079b79880}, 0xc026ebf180, 0xc054d239f0}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc087c49930, 0x63ca420}, {0x63caa08, 0xc03879f620}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:516 +0x147 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc087c49930, {0x63caa08, 0xc03879f620}, {0x4ed688c, 0x5}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c9 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc087c49930, {0x63caa08, 0xc03879f620}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x63c8510, 0xc079b79880}, 0xc026ebf180, 0xc054d239f0}, {0x63caa08, 0xc03879f620}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:519 +0x125 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc074022700, {0x63c8510, 0xc079b79880}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:146 +0x87 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:120 +0x34 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x200000000) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc0843d6ea0, {0x63c8510, 0xc079b79880}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:119 +0x6b github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc0843d6ea0, {0x63c8510, 0xc079b79880}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:753 +0x3a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc066366680, {0x63c8510, 0xc079b79880}, 0xc0843d6ea0) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:469 +0x258 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc054d23ce8, {0x63c8510, 0xc079b79880}, 0xc0385a9d90) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:303 +0x205 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc079b79140}, 0xc074582620, 0xc0385a9d90, 0xc0539d8d00, 0xc0ae234700, 0xc054d23ce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc079b79140}, 0xc054d239f0, 0xc074582620, 0xc06bcfc6c0, {{0x63caa08, 0xc03879f620}, 0x0}, 0xc0ae234700) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc054d23300, {0x63c8510, 0xc079b79140}, 0xc054d239f0, 0xc079b79140, {0x7fe42b790cf0, 0xc06bcfc6c0}, 0x0, 0xc044531318) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc054d23300, {0x63c8510, 0xc079b79140}, 0xc054d239f0, {0x7fe42b790cf0, 0xc06bcfc6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc054d23300, {0x63c85b8, 0xc027d9bfb0}, {{0x63f8778, 0xc02b172fa0}, {0xc0b043d303, 0x17d}, 0x14, 0x1}, 0xc061579560, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc054d23300, {0x63c85b8, 0xc027d9bfb0}, {{0x63f8778, 0xc02b172fa0}, {0xc0b043d303, 0x17d}, 0x14, 0x1}, 0xc061579560, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc054d23300, {0x63c85b8, 0xc027d9bfb0}, {0xc061579560, {0xc066ba3040, 0x14, 0x14}, {0x9bf64bc, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc084c54ea0, 0x0}, 0x0, {0x2c3771e1, 0xedb7a8350, 0x0}, 0x1}, 0xc054d23300, 0xc087c4b980, 0xc087c4b970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc054d23300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc054d23300, {0x63c8510, 0xc08d7a6200}, 0xc052b41dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08ce8ea80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc08d7a6200}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08ce8ea80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219794473 [runnable]: sync.runtime_SemacquireMutex(0xc050b68690, 0x8f, 0x7fdf35431e80) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc050b68538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc050b68560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc052ae7b00, {0xffffffffffffff15, 0xc050b686c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc052ae7b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc052ae7b00, {0xc06d77fcb0, 0x32, 0x8c}, {0xc09985c820, 0x32, 0x50}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07b009e98, {0xc06d77fcb0, 0x32, 0x8c}, {0xc09985c820, 0x32, 0x50}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07b0087c0, {0xc09985c820, 0x50, 0x8c}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07b0087c0, {0xc06d77fcb0, 0x50, 0xc09985c820}, {0xc09985c820, 0xc070abee60, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07b008500, {0xc09985c820, 0x32, 0x50}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04272ada8, {{0xc070abee60, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc05667af00, {{0xc052d83380, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04765c000, {0x63c85b8, 0xc057af1bf0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc057af1bf0}, {0x64a6560, 0xc05667af00}, {0xc052d83380, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc057af1bf0}, {0x6466ed8, 0xc04272ab00}, {0xc052d83380, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc026662690}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc058b9fb00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020076000, {0x63c85b8, 0xc057af1bf0}, 0xc058b9fb00, 0xc0415d08c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020076000, {0x63c85b8, 0xc057af1bf0}, 0xc058b9fb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057af1b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057af1b60}, {0xc07d2528f0, 0x660089}, 0xc07d2528f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057af1b60}, {0xc00102e4e0}, 0xc058b9fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057af1b00}, 0xc058b9fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc057af19b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791652 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc08230f980}, 0xc0387f7cb0, {0x646cc48, 0xc08230faa0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc08230f980}, 0xedb7a834e, 0xc0895ebb00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc08230f980}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0772dc330}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0387f7ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0772dc330}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801309 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc0707e88a0}, 0xc06549fd00, 0xc07de81680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0707e88a0}, 0xc06549fd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101bd6f1ae, 0x0, 0x0}, 0xc030ca2f70, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101bd6f1ae, 0x0, 0x0}, 0xc030ca2f70, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02878f380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02878f380}, {0xc07b0d52a8, 0x660089}, 0xc07b0d52b0) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02878f380}, {0xc00102e4e0}, 0xc01730c500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02878f320}, 0xc01730c500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8580, 0xc0273115c0}, 0x0, 0xc042e82b00) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c8580, 0xc0273115c0}, 0x63c8580) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftLogQueue).process(0xc003629840, {0x63c8580, 0xc0273115c0}, 0xc031bac800, {0xc0273115c0, 0x65f500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_log_queue.go:740 +0x87c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica.func1({0x63c8580, 0xc0273115c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:977 +0x26e github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc02878f110}, {0xc02a058330, 0x24}, 0xdf8475800, 0xc002eede28) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processReplica(0xc00d506f00, {0x63c85b8, 0xc02878f0e0}, {0x6463f50, 0xc031bac800}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:936 +0x3f5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2.1({0x63c85b8, 0xc02878f0e0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:855 +0xde github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805023 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc02956d860}, 0xc0269d7620, 0x1, 0x708017dc227036, 0x37e11d600, 0xc0577cf320, 0xc0587cc240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02956d860}, 0x1e0, 0xc07bd7d230, 0x577cf2c0, 0xc0bf12ed00, 0x18, 0xc0577cf320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc02956d860}, 0xc0577cf2c0, {{{{0x0, 0x0}, {0xc058243200, 0x56}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc02956d860}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc02956d860}, {0xc058aa4b40, {0x174301103f3ec9da, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02956d860}, 0xc07531ce60, 0xc07bd7d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02956d860}, 0x660089, {0xc058aa4b40, {0x174301103f3ec9da, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02956d860}, 0xc04ebb0000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f3ec9da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f3ec9da, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02956d650}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02956d650}, {0xc07bd7f3f8, 0x660089}, 0xc07bd7f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02956d650}, {0x0}, 0xc067ff5e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02956d530}, 0xc067ff5e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}, 0xc06612b020, 0xc0269d75f0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}, 0x3, 0xc06db174c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07bd7f980}}}, {0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}, 0xc06612b020, 0xc06db174c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02956d530}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02956d530}, {0x40, 0xc07bd7fb70}, 0xc07bd7fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02956d530}, {0x4e9d7a0, 0xc067ff5e00}, 0xc06612b020, 0xc0269d75f0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02956d530}, 0xc0275c9200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058aa46c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058aa46c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809759 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc04996d1a0}, 0xc0430a4978, 0x1, 0xc00374b0a8, 0x519bbc, 0xc087e06d20, 0xc035ced500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05ac9e7e0, {0x63c85b8, 0xc04996d1a0}, 0xc019e1a120, 0xc00374b1a8, 0x1c97ccf, 0xc084b505a0, 0x2, 0xc087e06d20, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc04996d1a0}, 0xc087e06c60, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04d9306c0, 0x85}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04996d1a0}, 0x6352260, 0x603b93f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc04996d1a0}, {0xc087e06b40, {0x174301102b7eff25, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc04996d1a0}, 0xc0622be460, 0xc00374b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04996d1a0}, 0x660089, {0xc087e06b40, {0x174301102b7eff25, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04996d1a0}, 0xc059bb1b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102b7eff25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102b7eff25, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04996d110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04996d110}, {0xc00374d3f8, 0x660089}, 0xc00374d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04996d110}, {0x7fe429376118}, 0xc059bb1900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04996d080}, 0xc059bb1900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}, 0xc07c185680, 0xc00c412b58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}, 0x3, 0xc050945780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00374d980}}}, {0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}, 0xc07c185680, 0xc050945780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04996d080}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04996d080}, {0x40, 0xc00374db70}, 0xc00374dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04996d080}, {0x4e9d7a0, 0xc059bb1900}, 0xc07c185680, 0xc00c412b58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04996d080}, 0xc082a8e300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc087e06a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc087e06a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812642 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219790589 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc05f452b40}, 0xc034634cb0, {0x646cc48, 0xc05f452ba0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc05f452b40}, 0xedb7a834e, 0xc03ecd0b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc05f452b40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06e426990}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc034634ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06e426990}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811325 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc0477b6150}, 0xc0301ea930, 0x1, 0xc0327090a8, 0x519bbc, 0xc066f0a180, 0xc094be84e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0477b6150}, 0x120, 0xc032709230, 0x66f0a120, 0x0, 0x660069, 0xc066f0a180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc0477b6150}, 0xc066f0a120, {{{{0x0, 0x0}, {0xc0ad9c06c0, 0x29}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0477b6150}, 0x6352410, 0x7818ffb0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc0477b6150}, {0xc066f0a000, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc0477b6150}, 0xc02044ebe0, 0xc0327099f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0477b6150}, 0x660089, {0xc066f0a000, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc0477b6150}, 0xc053873e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0477b60c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0477b60c0}, {0xc03270b3f8, 0x660089}, 0xc03270b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0477b60c0}, {0xc0015221e0}, 0xc053873c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0477b6000}, 0xc053873c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}, 0xc028222500, 0xc0beab8e40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}, 0x3, 0xc0b255c680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03270b980}}}, {0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}, 0xc028222500, 0xc0b255c680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0477b6000}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0477b6000}, {0x40, 0xc03270bb70}, 0xc03270bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0477b6000}, {0x4e9d7a0, 0xc053873c00}, 0xc028222500, 0xc0beab8e40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0477b6000}, 0xc043afb440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc015171d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc015171d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804984 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0302e3800, {0x63c85b8, 0xc073d400f0}, 0xc0496c6400, 0xc06828cdc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0302e3800, {0x63c85b8, 0xc073d400f0}, 0xc0496c6400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103eba85b3, 0x0, 0x0}, 0xc032c4eae0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103eba85b3, 0x0, 0x0}, 0xc032c4eae0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073d40060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d40060}, {0xc057c75918, 0x660089}, 0xc057c75920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073d40060}, {0xc00102e4e0}, 0xc0496c6300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a905f80}, 0xc0496c6300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc0cd726a20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03a905ef0}, {0xc00dfd36b0, 0x24}, 0x46c76f91d, 0xc028b32d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0xa}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc03a905ef0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810210 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0199c6bd0}, 0xc0273488b8, 0x1, 0xc00ffd30a8, 0x519bbc, 0xc08979db80, 0xc05a936540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0189dd560, {0x63c85b8, 0xc0199c6bd0}, 0xc0b4c8ed80, 0xc00ffd31a8, 0x8979db80, 0x0, 0x0, 0xc08979db80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0199c6bd0}, 0xc0218d69c0, {{{{0xc079db7b00, 0x46}, {0xc036b5e6c0, 0x7c}}, {{0xc0b3044000, 0x9a}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0199c6bd0}, 0x63521a0, 0x6b2455e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0199c6bd0}, {0xc04aa29e60, {0x1743011005545edf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0199c6bd0}, 0xc07076ad20, 0xc00ffd39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0199c6bd0}, 0x660089, {0xc04aa29e60, {0x1743011005545edf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0199c6bd0}, 0xc053812000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011005545edf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011005545edf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0199c6b40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0199c6b40}, {0xc00ffd53f8, 0x660089}, 0xc00ffd5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0199c6b40}, {0x7fe4257922e8}, 0xc06f82fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0199c6a80}, 0xc06f82fc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}, 0xc0701d4980, 0xc0bb3a9788) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}, 0x3, 0xc018d8ffc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00ffd5980}}}, {0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}, 0xc0701d4980, 0xc018d8ffc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0199c6a80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0199c6a80}, {0x40, 0xc00ffd5b70}, 0xc00ffd5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0199c6a80}, {0x4e9d7a0, 0xc06f82fc00}, 0xc0701d4980, 0xc0bb3a9788) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0199c6a80}, 0xc0218d67e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04aa29440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc04aa29440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807933 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc077516e70}, 0xc0506a6648, 0x1, 0x0, 0x1, 0xc0731415c0, 0xc099e0c4e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077516e70}, 0x120, 0xc00e81e728, 0x73141560, 0x0, 0x660069, 0xc0731415c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc077516e70}, 0xc073141560, {{{{0x0, 0x0}, {0xc03ce90480, 0x1b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077516e70}, 0x6352410, 0x32430ae8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc077516e70}, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc077516e70}, 0xc04c3980a0, 0xc07c606ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077516e70}, 0x660089, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc077516e70}, 0xc05156ad00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077516de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077516de0}, {0xc07d2508f0, 0x660089}, 0xc07d2508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077516de0}, {0xc00102e4e0}, 0xc05156ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077516d80}, 0xc05156ac00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04e72e810}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810134 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc076482540}, {0xc0aedd0160, 0x2, 0xc031b39e58}, {0x0, {0x1743010ff700c2e7, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc076482540}, 0x18701, 0xc022989560, 0x20, 0xc020362170) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc076482540}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811774 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64000, {0x63c85b8, 0xc04d7f6510}, 0xc0208f0c60, 0x1, 0xc033b190a8, 0x519bbc, 0xc0408093e0, 0xc08bd85ce0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc023bf7b00, {0x63c85b8, 0xc04d7f6510}, 0xc0408093e0, 0xc033b191a8, 0x40, 0x60, 0x0, 0xc0408093e0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64000, {0x63c85b8, 0xc04d7f6510}, 0xc040809320, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc033f8e6c0, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04d7f6510}, 0x6352260, 0x11fab470) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a588c0, {0x63c85b8, 0xc04d7f6510}, {0xc0408090e0, {0x174301103f87867f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a588c0, {0x63c85b8, 0xc04d7f6510}, 0xc05b6354a0, 0xc033b199f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04d7f6510}, 0x660089, {0xc0408090e0, {0x174301103f87867f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016ba0000, {0x63c85b8, 0xc04d7f6510}, 0xc082921d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f87867f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04d7f6480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d7f6480}, {0xc033b1b3f8, 0x660089}, 0xc033b1b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04d7f6480}, {0x7fe3c14f3e00}, 0xc082921b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04d7f63f0}, 0xc082921b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}, 0xc029ae9580, 0xc0208f17a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}, 0x3, 0xc05aca1480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc033b1b980}}}, {0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}, 0xc029ae9580, 0xc05aca1480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04d7f63f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04d7f63f0}, {0x40, 0xc033b1bb70}, 0xc033b1bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04d7f63f0}, {0x4e9d7a0, 0xc082921b00}, 0xc029ae9580, 0xc0208f17a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04d7f63f0}, 0xc0253127e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc040808fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc040808fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810324 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0307143c0, {0x63c85b8, 0xc07a44dd10}, 0xc0d1f770f8, 0x1, 0xc017a430a8, 0x519bbc, 0xc05f0bc180, 0xc05f91c060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07a44dd10}, 0x120, 0xc017a43230, 0x5f0bc120, 0x0, 0x660069, 0xc05f0bc180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0307143c0, {0x63c85b8, 0xc07a44dd10}, 0xc05f0bc120, {{{{0x0, 0x0}, {0xc03936f9e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07a44dd10}, 0x6352410, 0x3102fee0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0306f0820, {0x63c85b8, 0xc07a44dd10}, {0xc05f0bc000, {0x174301106b8c2d93, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0306f0820, {0x63c85b8, 0xc07a44dd10}, 0xc05ad9eb40, 0xc017a439f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07a44dd10}, 0x660089, {0xc05f0bc000, {0x174301106b8c2d93, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030720000, {0x63c85b8, 0xc07a44dd10}, 0xc064713f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106b8c2d93, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106b8c2d93, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a44dc80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a44dc80}, {0xc017a453f8, 0x660089}, 0xc017a45400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a44dc80}, {0x0}, 0xc064713c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a44dbf0}, 0xc064713c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}, 0xc088711ae0, 0xc047b81680) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}, 0x3, 0xc01d4b4440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017a45980}}}, {0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}, 0xc088711ae0, 0xc01d4b4440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a44dbf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a44dbf0}, {0x40, 0xc017a45b70}, 0xc017a45ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a44dbf0}, {0x4e9d7a0, 0xc064713c00}, 0xc088711ae0, 0xc047b81680) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a44dbf0}, 0xc02a2e0d20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0191e9e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0191e9e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810199 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0183a4c00, {0x63c85b8, 0xc04b447380}, 0xc089186300, 0xc0123ea320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0183a4c00, {0x63c85b8, 0xc04b447380}, 0xc089186300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011030193d3d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011030193d3d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04b4472c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04b4472c0}, {0xc01aafb3f8, 0x660089}, 0xc01aafb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04b4472c0}, {0x0}, 0xc089186100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04b447230}, 0xc089186100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}, 0xc01dddd2e0, 0xc0218490e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}, 0x3, 0xc08d631940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01aafb980}}}, {0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}, 0xc01dddd2e0, 0xc08d631940) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04b447230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04b447230}, {0x40, 0xc01aafbb70}, 0xc01aafbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04b447230}, {0x4e9d7a0, 0xc089186100}, 0xc01dddd2e0, 0xc0218490e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04b447230}, 0xc01edd5860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc011985d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc011985d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804978 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc025d797c0, {0x63c85b8, 0xc048383290}, 0xc0483f7380, 0x1, 0x708017d899a96c, 0x37e11d600, 0xc054937500, 0xc036758600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc048383290}, 0x1e0, 0xc07af9d230, 0x549374a0, 0xc06f76bba0, 0x18, 0xc054937500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc025d797c0, {0x63c85b8, 0xc048383290}, 0xc0549374a0, {{{{0x0, 0x0}, {0xc076750d80, 0x8}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc048383290}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc025d3d090, {0x63c85b8, 0xc048383290}, {0xc06f85d680, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc025d3d090, {0x63c85b8, 0xc048383290}, 0xc087686500, 0xc07af9d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc048383290}, 0x660089, {0xc06f85d680, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc025dbd400, {0x63c85b8, 0xc048383290}, 0xc011f4ec00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc048383200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048383200}, {0xc07af9f3f8, 0x660089}, 0xc07af9f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc048383200}, {0x4d939c}, 0xc011f4ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048383050}, 0xc011f4ea00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}, 0xc0235c5520, 0xc0483f7350) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}, 0x3, 0xc01ae27480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07af9f980}}}, {0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}, 0xc0235c5520, 0xc01ae27480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048383050}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048383050}, {0x40, 0xc07af9fb70}, 0xc07af9fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}, 0x203006, 0x203006) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048383050}, {0x4e9d7a0, 0xc011f4ea00}, 0xc0235c5520, 0xc0483f7350) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048383050}, 0xc027dfb260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85cfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85cfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808557 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc05fcdd6e0}, 0xc04784b638, 0x1, 0x708017fd44bff9, 0x37e11d600, 0xc066edd920, 0xc06c8d4de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05fcdd6e0}, 0x120, 0xc00bfd9230, 0x66edd8c0, 0x0, 0x660069, 0xc066edd920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc05fcdd6e0}, 0xc066edd8c0, {{{{0x0, 0x0}, {0xc046a7c480, 0x5b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05fcdd6e0}, 0x6352410, 0x479963c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc05fcdd6e0}, {0xc066edd7a0, {0x17430110602c44a8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc05fcdd6e0}, 0xc0c514e140, 0xc00bfd99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05fcdd6e0}, 0x660089, {0xc066edd7a0, {0x17430110602c44a8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc05fcdd6e0}, 0xc066f3bf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110602c44a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110602c44a8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05fcdd530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05fcdd530}, {0xc00bfdb3f8, 0x660089}, 0xc00bfdb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05fcdd530}, {0x7fe32e7aa1d8}, 0xc066f3bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05fcdd440}, 0xc066f3bd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}, 0xc01949e540, 0xc04784b608) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}, 0x3, 0xc052db5a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00bfdb980}}}, {0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}, 0xc01949e540, 0xc052db5a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05fcdd440}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05fcdd440}, {0x40, 0xc00bfdbb70}, 0xc00bfdbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05fcdd440}, {0x4e9d7a0, 0xc066f3bd00}, 0xc01949e540, 0xc04784b608) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05fcdd440}, 0xc077a76540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc066edd680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc066edd680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812386 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021def400, {0x63c85b8, 0xc073ef5080}, 0xc04aefccd8, 0x1, 0xc039db30a8, 0x519bbc, 0xc04f42fb60, 0xc09fd4d4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc073ef5080}, 0x120, 0xc039db3230, 0x4f42fb00, 0x0, 0x660069, 0xc04f42fb60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021def400, {0x63c85b8, 0xc073ef5080}, 0xc04f42fb00, {{{{0x0, 0x0}, {0xc03cfe86c0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073ef5080}, 0x6352410, 0x65f8ad48) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3dc20, {0x63c85b8, 0xc073ef5080}, {0xc04f42f9e0, {0x174301107e938c3a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3dc20, {0x63c85b8, 0xc073ef5080}, 0xc04e4cd220, 0xc039db39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073ef5080}, 0x660089, {0xc04f42f9e0, {0x174301107e938c3a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021ebd800, {0x63c85b8, 0xc073ef5080}, 0xc05eaf6e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e938c3a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e938c3a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073ef4ff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073ef4ff0}, {0xc039db53f8, 0x660089}, 0xc039db5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073ef4ff0}, {0x3333663732383230}, 0xc05eaf6c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073ef4f60}, 0xc05eaf6c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}, 0xc057022700, 0xc07a0d5c50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}, 0x3, 0xc046c7ed00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc039db5980}}}, {0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}, 0xc057022700, 0xc046c7ed00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073ef4f60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073ef4f60}, {0x40, 0xc039db5b70}, 0xc039db5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073ef4f60}, {0x4e9d7a0, 0xc05eaf6c00}, 0xc057022700, 0xc07a0d5c50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073ef4f60}, 0xc06270f980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04f42f8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04f42f8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812623 [semacquire]: sync.runtime_SemacquireMutex(0xc04e276400, 0xe0, 0x11840a5) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).Lock(0xc04278ee18) GOROOT/src/sync/rwmutex.go:116 +0x71 github.com/cockroachdb/pebble/internal/cache.(*shard).Set(0xc00074aa90, 0x11869e5, 0x7fd47f305b80, 0x7fb9, 0x7fe3c7921600) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:139 +0x85 github.com/cockroachdb/pebble/internal/cache.(*Cache).Set(0xc007a8b460, 0x7fd47f305b01, 0x7fd47f305ba0, 0x497f, 0x4984) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:750 +0x46 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc001d4e000, {0x16190cd, 0x497f}, 0x0, 0xc0c9cdb3f8) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2378 +0x637 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0c9cdb180, {0xffffffffffffff15, 0xc04278f0a0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0c9cdb180) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0c9cdb180, {0xc0c24eb0e0, 0x27, 0x4a}, 0xf, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekGE(0xc0923bc898, {0xc0c24eb0e0, 0x1, 0x4a}, 0x38) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:513 +0xaf github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc0923bc898, {0xc0c24eb0e0, 0x27, 0x4a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:409 +0xd6 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0923bb1c0, {0xc0c24eb0e0, 0x60, 0x4a}, 0x27, 0x27) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc0923bb1c0, {0xc0c24eb0e0, 0xc0c24eb0e0, 0xc07337fb00}, 0x4a) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc0923baf00, {0xc0c24eb0e0, 0x27, 0x4a}, {0x0, 0x0, 0x0}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/pebble.(*Iterator).SeekGE(...) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:865 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04c1012d8, {{0xc03d5757d0, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:290 +0xe9 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0626e8800, {{0xc03d5757d0, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc04fd74000, {{0xc03d5757d0, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc04fd74000, {0x63c85b8, 0xc02b87a7e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc03d575800, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.MVCCScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc03d575800, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2504 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc0d233b818}, {{0x1743011098ad09ea, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:62 +0x41b github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011098ad09ea, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc07314f700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b67a6eea, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02c293000, {0x63c85b8, 0xc02b87a7e0}, 0xc07314f700, 0xc0822517c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c293000, {0x63c85b8, 0xc02b87a7e0}, 0xc07314f700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011098ad09ea, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011098ad09ea, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02b87a510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b87a510}, {0xc0427953f8, 0x660089}, 0xc042795400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02b87a510}, {0xd0050000100a0000}, 0xc07314f500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02b87a480}, 0xc07314f500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}, 0xc019cca0e0, 0xc060f8e840) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}, 0x3, 0xc015c8e480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc042795980}}}, {0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}, 0xc019cca0e0, 0xc015c8e480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02b87a480}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02b87a480}, {0x40, 0xc042795b70}, 0xc042795ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02b87a480}, {0x4e9d7a0, 0xc07314f500}, 0xc019cca0e0, 0xc060f8e840) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02b87a480}, 0xc08bd8b920, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356aea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc05356aea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801431 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc088338548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0883385b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc092f2d1f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc082a19180, {0xc022825f80, 0x2b, 0x80}, {0xc0da1d88c0, 0x2b, 0x6d}, 0x1, 0xd1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092f2d088, {0xc022825f80, 0x1, 0x6d}, {0xc0da1d88c0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092f2d088, {0xc022825f80, 0x2b, 0x80}, {0xc0da1d88c0, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092f2c7c0, {0xc0da1d88c0, 0x6d, 0x80}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092f2c7c0, {0xc022825f80, 0x2b, 0xc0da1d88c0}, {0xc0da1d88c0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092f2c500, {0xc0da1d88c0, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06bc4ac08, {{0xc0613b4a50, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04da77a00, {{0xc0613b4a50, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc059f3d800, {0x63c85b8, 0xc046ee8810}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc046ee8810}, {0x64a6560, 0xc04da77a00}, {0xc0613b4a50, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc046ee8810}, {0x6466ed8, 0xc06bc4ac00}, {0xc0613b4a50, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01b5efa70}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc038b18500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c92400, {0x63c85b8, 0xc046ee8810}, 0xc038b18500, 0xc08f98af00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c92400, {0x63c85b8, 0xc046ee8810}, 0xc038b18500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046ee8780}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046ee8780}, {0xc0936948f0, 0x660089}, 0xc0936948f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046ee8780}, {0xc00102e4e0}, 0xc04859ff00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046ee8720}, 0xc04859ff00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02ef022a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219795202 [runnable]: sync.runtime_SemacquireMutex(0xc005b3d198, 0x8f, 0x7fe19f73fa40) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc005b3d040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc005b3d068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0c35ca000, {0xffffffffffffff15, 0xc005b3d1c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0c35ca000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0c35ca000, {0xc02e844580, 0x32, 0x74}, {0xc087dc2780, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc03f30e898, {0xc02e844580, 0x32, 0x74}, {0xc087dc2780, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc03f30d1c0, {0xc087dc2780, 0x77, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc03f30d1c0, {0xc02e844580, 0x77, 0xc087dc2780}, {0xc087dc2780, 0xc055f4f140, 0x31}, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc03f30cf00, {0xc087dc2780, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc086b87428, {{0xc055f4f140, 0x31, 0xc0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04fb25b00, {{0xc03f0679e0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07e2c7400, {0x63c85b8, 0xc0400d9d40}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0400d9d40}, {0x64a6560, 0xc04fb25b00}, {0xc03f0679e0, 0x0, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0400d9d40}, {0x6466ed8, 0xc086b87180}, {0xc03f0679e0, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc074391e90}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06e0f4500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b0f800, {0x63c85b8, 0xc0400d9d40}, 0xc06e0f4500, 0xc046dec960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b0f800, {0x63c85b8, 0xc0400d9d40}, 0xc06e0f4500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0400d9c80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0400d9c80}, {0xc005b433f8, 0x660089}, 0xc005b43400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0400d9c80}, {0xc0015221e0}, 0xc02b8d7f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0400d9bf0}, 0xc02b8d7f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}, 0xc0291cbda0, 0xc0225af2d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}, 0x3, 0xc054d93a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005b43980}}}, {0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}, 0xc0291cbda0, 0xc054d93a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0400d9bf0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0400d9bf0}, {0x40, 0xc005b43b70}, 0xc005b43ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0400d9bf0}, {0x4e9d7a0, 0xc02b8d7f00}, 0xc0291cbda0, 0xc0225af2d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0400d9bf0}, 0xc06af57740, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0153c6900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0153c6900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808759 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024fae280, {0x63c85b8, 0xc06b798690}, 0xc030342a80, 0x1, 0xc01053f0a8, 0x519bbc, 0xc086e747e0, 0xc03621d8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06b798690}, 0x1e0, 0xc01053f230, 0x86e74780, 0x0, 0x660069, 0xc086e747e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024fae280, {0x63c85b8, 0xc06b798690}, 0xc086e74780, {{{{0x0, 0x0}, {0xc04c660d80, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06b798690}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024ef1360, {0x63c85b8, 0xc06b798690}, {0xc062af65a0, {0x174301105e8c1334, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024ef1360, {0x63c85b8, 0xc06b798690}, 0xc0cd204aa0, 0xc01053f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06b798690}, 0x660089, {0xc062af65a0, {0x174301105e8c1334, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024fb7800, {0x63c85b8, 0xc06b798690}, 0xc06d523200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e8c1334, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e8c1334, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b798600}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b798600}, {0xc0105413f8, 0x660089}, 0xc010541400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b798600}, {0x7fe34bb2c778}, 0xc06d523000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06b798570}, 0xc06d523000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}, 0xc078c6b8c0, 0xc0418adef0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}, 0x3, 0xc02cdbb380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010541980}}}, {0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}, 0xc078c6b8c0, 0xc02cdbb380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06b798570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b798570}, {0x40, 0xc010541b70}, 0xc010541ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06b798570}, {0x4e9d7a0, 0xc06d523000}, 0xc078c6b8c0, 0xc0418adef0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06b798570}, 0xc0260610e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc062af6480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc062af6480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804894 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc06786a570}, 0xc0d04a12a8, 0x1, 0xc08850d0a8, 0x519bbc, 0xc055f2de60, 0xc04c2b4f60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06786a570}, 0x1e0, 0xc08850d230, 0x55f2de00, 0xc04ab36ea0, 0x18, 0xc055f2de60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc06786a570}, 0xc055f2de00, {{{{0x0, 0x0}, {0xc058da0000, 0x3d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc06786a570}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc06786a570}, {0xc0583557a0, {0x17430110311c7881, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc06786a570}, 0xc065e2b040, 0xc08850d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06786a570}, 0x660089, {0xc0583557a0, {0x17430110311c7881, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc06786a570}, 0xc028dbdf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110311c7881, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110311c7881, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06786a4e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06786a4e0}, {0xc08850f3f8, 0x660089}, 0xc08850f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06786a4e0}, {0xc0015221e0}, 0xc028dbdd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06786a450}, 0xc028dbdd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}, 0xc0b85a7960, 0xc054314a50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}, 0x3, 0xc060d4dec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc08850f980}}}, {0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}, 0xc0b85a7960, 0xc060d4dec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06786a450}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06786a450}, {0x40, 0xc08850fb70}, 0xc08850fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}, 0x203018, 0x203018) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06786a450}, {0x4e9d7a0, 0xc028dbdd00}, 0xc0b85a7960, 0xc054314a50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06786a450}, 0xc04c3b37a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058355680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058355680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219769605 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a1710, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc02881ff00, 0xc00b924800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc02881ff00, {0xc00b924800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc02881ff00, {0xc00b924800, 0x735, 0xc05793cb40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03a196940, {0xc00b924800, 0x7eda79, 0xc00b924805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0707b9ce0, {0xc00b924800, 0x174301107aeaabed, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0b2691e00, {0xc00b924800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0643ea978, {0x631b900, 0xc0b2691e00}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0643ea700, {0x7fe42ab59310, 0xc0707b9ce0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0643ea700, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0643ea700, {0xc00320c000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0445245d0, {0xc00320c000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc04d6beb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc04d6beb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc04d6bf0e0, {0x634aac0, 0xc04d6beb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc04d6bea80, 0xc02c19cb00, 0xc08317342b, 0xc08317342d, {0x635bec0, 0xc03bbae000}, 0xc000d6ae00, 0xc0629a22d8, 0xc083173429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc04d6bea80, {0x63c85b8, 0xc0740515f0}, 0xc02c19caf0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc07dbcc037, 0x8}}, 0x0, 0xc074051590, 0xc0740515c0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0740515f0}, {0x643d810, 0xc0707b9ce0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07b8c6480}, {0x643d810, 0xc0707b9ce0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07b8c6480}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812548 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x80, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc04b7d0a80, 0xc034a4b168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04b7d0a80, {0xc007e5f5e0, 0x30, 0x6b}, {0xc01e2dabd0, 0x30, 0x6f}, 0x0, 0x3b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092193a88, {0xc007e5f5e0, 0x1, 0x6f}, {0xc01e2dabd0, 0xc092192f00, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092193a88, {0xc007e5f5e0, 0x30, 0x6b}, {0xc01e2dabd0, 0x30, 0x6f}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0921931c0, {0xc01e2dabd0, 0x0, 0xc034a4b4e0}, 0x121ea30, 0xd0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0921931c0, {0xc007e5f5e0, 0x30, 0x6f}, {0xc01e2dabd0, 0x2f, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092192f00, {0xc01e2dabd0, 0x30, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc051d5a008, {{0xc03a992b40, 0x2f, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06583b100, {{0xc03a992b40, 0x2f, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04765d800, {0x63c85b8, 0xc056614060}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056614060}, {0x64a6560, 0xc06583b100}, {0xc03a992b40, 0xc00bc1fa40, 0x30}, {0x1743011089a4011a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056614060}, {0x6466ed8, 0xc051d5a000}, {0xc03a992b40, 0x2f, 0x30}, {0x1743011089a4011a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc04621a420}, {{0x1743011089a4011a, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011089a4011a, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc083616c00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a771661a, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0219b8000, {0x63c85b8, 0xc056614060}, 0xc083616c00, 0xc02d48c460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc056614060}, 0xc083616c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011089a4011a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011089a4011a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04bf65fb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf65fb0}, {0xc034a513f8, 0x660089}, 0xc034a51400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04bf65fb0}, {0xc0015221e0}, 0xc083616a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04bf65f20}, 0xc083616a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}, 0xc0ccf7faa0, 0xc038161b18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}, 0x3, 0xc071e54780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034a51980}}}, {0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}, 0xc0ccf7faa0, 0xc071e54780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04bf65f20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04bf65f20}, {0x40, 0xc034a51b70}, 0xc034a51ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04bf65f20}, {0x4e9d7a0, 0xc083616a00}, 0xc0ccf7faa0, 0xc038161b18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04bf65f20}, 0xc05c2fbe00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea2480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803752 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02db26000, {0x63c85b8, 0xc045e0e8a0}, 0xc05977c000, 0xc079592280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02db26000, {0x63c85b8, 0xc045e0e8a0}, 0xc05977c000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ef14733, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x174301102ef113c7, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:514 +0x868 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func2({0x63c85b8, 0xc045e0e8a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:365 +0x12d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812709 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc02557b2c0}, 0xc05707ff00, 0xc0761ed7c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc02557b2c0}, 0xc05707ff00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c781366, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c781366, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02557b110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02557b110}, {0xc00daa33f8, 0x660089}, 0xc00daa3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02557b110}, {0x7fe3316ccbe0}, 0xc05707fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02557ae40}, 0xc05707fd00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}, 0xc00abd2620, 0xc047919908) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}, 0x3, 0xc01352a980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00daa3980}}}, {0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}, 0xc00abd2620, 0xc01352a980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02557ae40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02557ae40}, {0x40, 0xc00daa3b70}, 0xc00daa3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02557ae40}, {0x4e9d7a0, 0xc05707fd00}, 0xc00abd2620, 0xc047919908) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02557ae40}, 0xc03df19500, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e99e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0680e99e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812347 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc06454d140}, 0xc09dc1a468, 0x1, 0xc03a1b90a8, 0x519bbc, 0xc08aeb04e0, 0xc04cdc5b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc071f4dec0, {0x63c85b8, 0xc06454d140}, 0xc05159eb40, 0xc03a1b91a8, 0x8aeb0540, 0x0, 0x0, 0xc08aeb04e0, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc06454d140}, 0xc08aeb0480, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc06de018c0, 0xaf}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc06454d140}, 0x63523b0, 0x42012d58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc06454d140}, {0x0, {0x1743011081bcd5e1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc06454d140}, 0xc022d90fa0, 0xc03a1b99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06454d140}, 0x660089, {0x0, {0x1743011081bcd5e1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc06454d140}, 0xc0435dc200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011081bcd5e1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011081bcd5e1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011081bcd5e1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06454d0b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06454d0b0}, {0xc03a1bb3f8, 0x660089}, 0xc03a1bb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06454d0b0}, {0x7fe3c01cd2e8}, 0xc073561f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06454d020}, 0xc073561f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}, 0xc02dda7860, 0xc09dc1b050) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}, 0x3, 0xc0c35fad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03a1bb980}}}, {0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}, 0xc02dda7860, 0xc0c35fad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06454d020}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06454d020}, {0x40, 0xc03a1bbb70}, 0xc03a1bbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06454d020}, {0x4e9d7a0, 0xc073561f00}, 0xc02dda7860, 0xc09dc1b050) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06454d020}, 0xc07b1575c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08aeb0360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc08aeb0360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219787336 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a2070, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc079dfa980, 0xc064d33000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc079dfa980, {0xc064d33000, 0x73d, 0x73d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc079dfa980, {0xc064d33000, 0x73d, 0xc0524ca280}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0304bb4b8, {0xc064d33000, 0x7eda79, 0xc064d33005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04601a900, {0xc064d33000, 0xc0821eec60, 0x73d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03c545908, {0xc064d33000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0ae234cf8, {0x631b900, 0xc03c545908}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0ae234a80, {0x7fe42ab59310, 0xc04601a900}, 0x73d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0ae234a80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0ae234a80, {0xc08e62c000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc07349c030, {0xc08e62c000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0be1c1588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0be1c1588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0be1c1b60, {0x634aac0, 0xc0be1c1588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0be1c1500, 0xc04b4bd4e0, 0xc0962a342b, 0xc0962a342d, {0x635bec0, 0xc01b6fcab0}, 0xc000d6ae00, 0xc01ca54c64, 0xc0962a3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0be1c1500, {0x63c85b8, 0xc02a43ffb0}, 0xc04b4bd4d0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc0b1013009, 0x10}}, 0x0, 0xc02a43ff50, 0xc02a43ff80, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc02a43ffb0}, {0x643d810, 0xc04601a900}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc07275c4e0}, {0x643d810, 0xc04601a900}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc07275c4e0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804131 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01804b400, {0x63c85b8, 0xc024c1a1b0}, 0xc0647dea00, 0xc06068ca00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01804b400, {0x63c85b8, 0xc024c1a1b0}, 0xc0647dea00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032796b85, 0x0, 0x0}, 0xc061d8bc00, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032796b85, 0x0, 0x0}, 0xc061d8bc00, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024c1a060}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024c1a060}, {0xc076027918, 0x660089}, 0xc076027920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024c1a060}, {0xc00102e4e0}, 0xc0647de800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05c81bfb0}, 0xc0647de800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc020fde240}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc05c81bf20}, {0xc00dfd36b0, 0x24}, 0x46c79be29, 0xc00304fd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc03eaaea50}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc05c81bf20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812460 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e2d8000, {0x63c85b8, 0xc02e033320}, 0xc07ae4ec00, 0xc0761ed360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc02e033320}, 0xc07ae4ec00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c5c6ea6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c5c6ea6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02e032ff0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02e032ff0}, {0xc07ca69a28, 0x660089}, 0xc07ca69a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02e032ff0}, {0xc00102e4e0}, 0xc07ae4eb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02e032b10}, 0xc07ae4eb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0925b77d0, {0x63c85b8, 0xc02e032930}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0925b77d0, {0x63c85b8, 0xc02e032930}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc02e032930}, {0x4f67ad2, 0x6446ff8}, 0xc034637fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219802683 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fe85000, {0x63c85b8, 0xc086bba6f0}, 0xc0a38f8500, 0xc022e2b7c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fe85000, {0x63c85b8, 0xc086bba6f0}, 0xc0a38f8500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110248757c5, 0x0, 0x0}, 0xc0572a9e30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110248757c5, 0x0, 0x0}, 0xc0572a9e30, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc086bba660}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086bba660}, {0xc0501b93f8, 0x660089}, 0xc0501b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc086bba660}, {0x0}, 0xc0a38f8200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc086bba5d0}, 0xc0a38f8200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}, 0xc0270b5cc0, 0xc042d4a708) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}, 0x3, 0xc05511a380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0501b9980}}}, {0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}, 0xc0270b5cc0, 0xc05511a380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc086bba5d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc086bba5d0}, {0x40, 0xc0501b9b70}, 0xc0501b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc086bba5d0}, {0x4e9d7a0, 0xc0a38f8200}, 0xc0270b5cc0, 0xc042d4a708) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc086bba5d0}, 0xc030802000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652ca20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652ca20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809231 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6000, {0x63c85b8, 0xc01618c8a0}, 0xc07cc2b590, 0x1, 0xc01fa210a8, 0x519bbc, 0xc0565485c0, 0xc05f421f20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0555db7a0, {0x63c85b8, 0xc01618c8a0}, 0xc0565485c0, 0xc01fa211a8, 0x41, 0x60, 0x4, 0xc0565485c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6000, {0x63c85b8, 0xc01618c8a0}, 0xc056548380, {{{{0xc03eb11200, 0x6}, {0xc03cc52480, 0x4}}, {{0xc051fec120, 0xd}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01618c8a0}, 0x63521a0, 0x42a0f030) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddb30, {0x63c85b8, 0xc01618c8a0}, {0xc05330b200, {0x174301100a3cfddc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddb30, {0x63c85b8, 0xc01618c8a0}, 0xc06b94a280, 0xc01fa219f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01618c8a0}, 0x660089, {0xc05330b200, {0x174301100a3cfddc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc01618c8a0}, 0xc0523d4e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100a3cfddc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100a3cfddc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01618c810}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01618c810}, {0xc01fa233f8, 0x660089}, 0xc01fa23400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01618c810}, {0xc0015221e0}, 0xc0523d4b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01618c690}, 0xc0523d4b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}, 0xc02d9d6c60, 0xc043945248) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}, 0x3, 0xc03f78b780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fa23980}}}, {0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}, 0xc02d9d6c60, 0xc03f78b780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01618c690}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01618c690}, {0x40, 0xc01fa23b70}, 0xc01fa23ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01618c690}, {0x4e9d7a0, 0xc0523d4b00}, 0xc02d9d6c60, 0xc043945248) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01618c690}, 0xc0242a3680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05330b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802601 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc02de80120}, 0xc06b9a2300, 0xc05f31cd20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc02de80120}, 0xc06b9a2300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011023470b9d, 0x0, 0x0}, 0xc0100910b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011023470b9d, 0x0, 0x0}, 0xc0100910b0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02de80030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02de80030}, {0xc0629af3f8, 0x660089}, 0xc0629af400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02de80030}, {0x7fe3c15dba80}, 0xc0995adf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a435f50}, 0xc0995adf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}, 0xc05f206d60, 0xc04ccd42a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}, 0x3, 0xc01ab7c3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0629af980}}}, {0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}, 0xc05f206d60, 0xc01ab7c3c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a435f50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a435f50}, {0x40, 0xc0629afb70}, 0xc0629afab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a435f50}, {0x4e9d7a0, 0xc0995adf00}, 0xc05f206d60, 0xc04ccd42a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a435f50}, 0xc01b70a600, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0530c86c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0530c86c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807352 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02b26c000, {0x63c85b8, 0xc064c1e750}, 0xc08ce9dd00, 0xc0647d3860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc064c1e750}, 0xc08ce9dd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105709f616, 0x0, 0x0}, 0xc090a20480, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105709f616, 0x0, 0x0}, 0xc090a20480, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc064c1e6c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064c1e6c0}, {0xc00561d3f8, 0x660089}, 0xc00561d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc064c1e6c0}, {0xc04ed7c140}, 0xc08ce9db00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc064c1e630}, 0xc08ce9db00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}, 0xc072fd3380, 0xc036d66180) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}, 0x3, 0xc0615a0dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00561d980}}}, {0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}, 0xc072fd3380, 0xc0615a0dc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc064c1e630}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc064c1e630}, {0x40, 0xc00561db70}, 0xc00561dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc064c1e630}, {0x4e9d7a0, 0xc08ce9db00}, 0xc072fd3380, 0xc036d66180) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc064c1e630}, 0xc058f54900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc084aaa240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc084aaa240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812525 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812754 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802718 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02532b400, {0x63c85b8, 0xc030902780}, 0xc04f30a100, 0xc022927400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc030902780}, 0xc04f30a100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110255f691e, 0x0, 0x0}, 0xc064845520, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110255f691e, 0x0, 0x0}, 0xc064845520, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc030902390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc030902390}, {0xc05ff453f8, 0x660089}, 0xc05ff45400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc030902390}, {0x7fe13e89ce98}, 0xc0a67c9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0309022d0}, 0xc0a67c9f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}, 0xc04563a760, 0xc0b6a0fd40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}, 0x3, 0xc033390540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05ff45980}}}, {0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}, 0xc04563a760, 0xc033390540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0309022d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0309022d0}, {0x40, 0xc05ff45b70}, 0xc05ff45ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0309022d0}, {0x4e9d7a0, 0xc0a67c9f00}, 0xc04563a760, 0xc0b6a0fd40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0309022d0}, 0xc022633aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0599b58c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0599b58c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808346 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020f4dc00, {0x63c85b8, 0xc05b604f60}, 0xc06686ab00, 0xc041b80640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4dc00, {0x63c85b8, 0xc05b604f60}, 0xc06686ab00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c214c9f, 0x0, 0x0}, 0xc04ebdcc70, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c214c9f, 0x0, 0x0}, 0xc04ebdcc70, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05b604e70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05b604e70}, {0xc076023918, 0x660089}, 0xc076023920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05b604e70}, {0xc00102e4e0}, 0xc06686aa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05b604e10}, 0xc06686aa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xb, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc093edacc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc05b604d80}, {0xc00dfd36b0, 0x24}, 0x46c7b5cc6, 0xc0209a1d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x4c1af3}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc05b604d80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219632861 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc02b366190, 0xa6) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0cbf98000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0cbf98660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0594a8c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0594a8c00, {0x63c8510, 0xc02b3661c0}, 0xc07de95dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07d052000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc02b3661c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07d052000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812649 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810174 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fc5ca00, {0x63c85b8, 0xc0853c3a70}, 0xc038af9320, 0x1, 0xc021f710a8, 0x519bbc, 0xc0612eac00, 0xc04d525bc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc076750900, {0x63c85b8, 0xc0853c3a70}, 0xc0612eac00, 0xc021f711a8, 0x31, 0x40, 0x0, 0xc0612eac00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fc5ca00, {0x63c85b8, 0xc0853c3a70}, 0xc0612eab40, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc073a03e60, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0853c3a70}, 0x6352260, 0x50b34540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fc6a3c0, {0x63c85b8, 0xc0853c3a70}, {0xc0612eaa20, {0x174301103074bebc, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fc6a3c0, {0x63c85b8, 0xc0853c3a70}, 0xc089100280, 0xc021f719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0853c3a70}, 0x660089, {0xc0612eaa20, {0x174301103074bebc, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fc82000, {0x63c85b8, 0xc0853c3a70}, 0xc0cd522f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103074bebc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103074bebc, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0853c39e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0853c39e0}, {0xc021f733f8, 0x660089}, 0xc021f73400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0853c39e0}, {0x7fe3c2f17358}, 0xc0cd522d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0853c38c0}, 0xc0cd522d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}, 0xc02299c880, 0xc029398ed0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}, 0x3, 0xc07b3d5500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc021f73980}}}, {0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}, 0xc02299c880, 0xc07b3d5500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0853c38c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0853c38c0}, {0x40, 0xc021f73b70}, 0xc021f73ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0853c38c0}, {0x4e9d7a0, 0xc0cd522d00}, 0xc02299c880, 0xc029398ed0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0853c38c0}, 0xc071a90120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0578799e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0578799e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219781627 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc06d0dc370, {0x63c85b8, 0xc046bf6c00}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc082e88000, {0x63c85b8, 0xc046bf6c00}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc082e88000, {0x63c8510, 0xc01b621080}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc082e88000, {0x63c8510, 0xc01b621080}, {0x63f7db8, 0x9bf6f48}, 0xc0990dd088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc082e88000, {0x63c85b8, 0xc06f157f80}, {{0x63f7db8, 0x9bf6f48}, {0xc0ee4d04c6, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc082e88000, {0x63c85b8, 0xc06f157f80}, {{0x63f7db8, 0x9bf6f48}, {0xc0ee4d04c6, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0ee4d04c6, 0x6}, 0x0, 0x0}, {0x665743f, 0xedb7a834f, 0x0}, {0x6657bd9, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc082e88000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc082e88000, {0x63c8510, 0xc05121d380}, 0xc06b41ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc00f653b90) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc05121d380}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc00f653b90) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811667 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021dee500, {0x63c85b8, 0xc04f3589c0}, 0xc06292f908, 0x1, 0xc01b78b0a8, 0x519bbc, 0xc035a29ec0, 0xc044201260) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04f3589c0}, 0x120, 0xc01b78b230, 0x35a29e60, 0x0, 0x660069, 0xc035a29ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021dee500, {0x63c85b8, 0xc04f3589c0}, 0xc035a29e60, {{{{0x0, 0x0}, {0xc0783fc000, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f3589c0}, 0x6352410, 0x3f55a7a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3d4a0, {0x63c85b8, 0xc04f3589c0}, {0xc035a29d40, {0x1743011074421028, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3d4a0, {0x63c85b8, 0xc04f3589c0}, 0xc01c6975e0, 0xc01b78b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f3589c0}, 0x660089, {0xc035a29d40, {0x1743011074421028, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021de4800, {0x63c85b8, 0xc04f3589c0}, 0xc06e150500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011074421028, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011074421028, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f358930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f358930}, {0xc01b78d3f8, 0x660089}, 0xc01b78d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f358930}, {0x7fe3b59597c8}, 0xc06e150300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f3588a0}, 0xc06e150300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}, 0xc0867284c0, 0xc04170e888) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}, 0x3, 0xc0024fd880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01b78d980}}}, {0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}, 0xc0867284c0, 0xc0024fd880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f3588a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f3588a0}, {0x40, 0xc01b78db70}, 0xc01b78dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f3588a0}, {0x4e9d7a0, 0xc06e150300}, 0xc0867284c0, 0xc04170e888) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f3588a0}, 0xc028e950e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc035a29c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc035a29c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801983 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc031929c20}, 0xc03cca8300, 0xc074cfc460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc031929c20}, 0xc03cca8300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101f612a72, 0x0, 0x0}, 0xc0535cbe40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101f612a72, 0x0, 0x0}, 0xc0535cbe40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031929b60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031929b60}, {0xc0536793f8, 0x660089}, 0xc053679400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031929b60}, {0x0}, 0xc03cca8100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031929ad0}, 0xc03cca8100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}, 0xc085436180, 0xc03b526708) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}, 0x3, 0xc0c945c400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc053679980}}}, {0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}, 0xc085436180, 0xc0c945c400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc031929ad0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031929ad0}, {0x40, 0xc053679b70}, 0xc053679ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc031929ad0}, {0x4e9d7a0, 0xc03cca8100}, 0xc085436180, 0xc03b526708) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc031929ad0}, 0xc04d489560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc049c96000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc049c96000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804986 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02a0b7470}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807573 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e06c800, {0x63c85b8, 0xc01c3568d0}, 0xc098b3e100, 0xc075bfdc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc01c3568d0}, 0xc098b3e100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101cb07048, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101cb07048, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01c356390}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01c356390}, {0xc070135a28, 0x660089}, 0xc070135a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01c356390}, {0x24}, 0xc098b3e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01c3561e0}, 0xc098b3e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc0570ee850, {0x63c85b8, 0xc04f1e3f80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0570ee850, {0x63c85b8, 0xc04f1e3f80}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc04f1e3f80}, {0x4f67ad2, 0x6446ff8}, 0xc01526bfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219800980 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc018268400, {0x63c85b8, 0xc05a1791a0}, 0xc06b402a00, 0xc0767557c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018268400, {0x63c85b8, 0xc05a1791a0}, 0xc06b402a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011016e1f69e, 0x0, 0x0}, 0xc014f353a0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011016e1f69e, 0x0, 0x0}, 0xc014f353a0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05a179110}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05a179110}, {0xc076023918, 0x660089}, 0xc076023920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05a179110}, {0xc00102e4e0}, 0xc06b402900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05a179080}, 0xc06b402900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc0453d0180}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc05a178fc0}, {0xc00dfd36b0, 0x24}, 0x4a466d954, 0xc01f3f2d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x4c1af3}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc05a178fc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219792172 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc04ba615c0}, 0xc00a5bdcb0, {0x646cc48, 0xc04ba61620}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc04ba615c0}, 0xedb7a834e, 0xc08c785320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc04ba615c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03c7fe0c0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc00a5bdef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc03c7fe0c0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809021 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e034f00, {0x63c85b8, 0xc04ca40ea0}, 0xc0403c17e8, 0x1, 0xc0830ab920, 0x519bbc, 0xc02ac9cde0, 0xc078b9e3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc03ce90c60, {0x63c85b8, 0xc04ca40ea0}, 0xc03ce90c98, 0xc0830aba20, 0x41, 0x60, 0xffffffffffffffff, 0xc02ac9cde0, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e034f00, {0x63c85b8, 0xc04ca40ea0}, 0xc02ac9cd80, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0b4c8f8c0, 0x9}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc04ca40ea0}, 0x63523b0, 0x30992628) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd310, {0x63c85b8, 0xc04ca40ea0}, {0x0, {0x174301106e846c2b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd310, {0x63c85b8, 0xc04ca40ea0}, 0xc03f4cf180, 0xc0830ac268) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ca40ea0}, 0x660089, {0x0, {0x174301106e846c2b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc04ca40ea0}, 0xc088ab8f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ca40e10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ca40e10}, {0xc019c5fc70, 0x660089}, 0xc019c5fc78) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ca40e10}, {0xc047147c20}, 0xc088ab8e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ca40db0}, 0xc088ab8e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x174301106e846c2b, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) github.com/cockroachdb/cockroach/pkg/kv/db.go:968 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c85b8, 0xc070cc45d0}, 0x0, 0xc051882100) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc01876a5a0, {0x63c85b8, 0xc070cc45d0}, 0x19) github.com/cockroachdb/cockroach/pkg/kv/db.go:853 +0x74 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).MaybePushTransactions(0xc0078dea50, {0x63c85b8, 0xc070cc45d0}, 0xc019c626b0, {{0x174301106e8454ff, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:392 +0x6fb github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).PushTransaction(0x0, {0x63c85b8, 0xc070cc45d0}, 0xc077d28398, {{0x174301106e8454ff, 0x0, 0x0}, 0x0, {0x0, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:300 +0x165 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).pushLockTxn(0xc02dfdd360, {0x63c85b8, 0xc070cc45d0}, {0xc06be41d40, {0x1743011061c683f4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:499 +0x533 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn.func3({0x63c85b8, 0xc070cc45d0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:377 +0x285 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*lockTableWaiterImpl).WaitOn(0xc02dfdd360, {0x63c85b8, 0xc070cc45d0}, {0xc06be41d40, {0x1743011061c683f4, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock_table_waiter.go:428 +0x4a7 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd310, {0x63c85b8, 0xc070cc45d0}, 0xc0225cf720, 0xc019c63558) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:328 +0x94f github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc070cc45d0}, 0xc0225cf720, {0xc06be41d40, {0x1743011061c683f4, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e04d400, {0x63c85b8, 0xc070cc45d0}, 0xc06ecc8500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011061c683f4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011061c683f4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070cc4540}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070cc4540}, {0xc019c653f8, 0x660089}, 0xc019c65400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070cc4540}, {0x0}, 0xc06ecc8300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070cc44b0}, 0xc06ecc8300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}, 0xc0707cf100, 0xc05eae1a10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}, 0x3, 0xc0636ca140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc019c65980}}}, {0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}, 0xc0707cf100, 0xc0636ca140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070cc44b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070cc44b0}, {0x40, 0xc019c65b70}, 0xc019c65ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070cc44b0}, {0x4e9d7a0, 0xc06ecc8300}, 0xc0707cf100, 0xc05eae1a10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070cc44b0}, 0xc054015e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0500f4d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0500f4d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801969 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01804b400, {0x63c85b8, 0xc0ae218db0}, 0xc06cd05200, 0xc051624c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01804b400, {0x63c85b8, 0xc0ae218db0}, 0xc06cd05200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff4bd677e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff4bd677e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x9, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0ae218d20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ae218d20}, {0xc01a6b33f8, 0x660089}, 0xc01a6b3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0ae218d20}, {0xc0015221e0}, 0xc06cd04f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0ae218c30}, 0xc06cd04f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}, 0xc0621ffc40, 0xc064234930) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}, 0x3, 0xc07d320500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a6b3980}}}, {0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}, 0xc0621ffc40, 0xc07d320500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0ae218c30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0ae218c30}, {0x40, 0xc01a6b3b70}, 0xc01a6b3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0ae218c30}, {0x4e9d7a0, 0xc06cd04f00}, 0xc0621ffc40, 0xc064234930) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0ae218c30}, 0xc027453680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04cb0fb00, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04cb0fb00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219794610 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc07ac66548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc07ac665b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0a95eecf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc06b21c000, {0xc0d84f8600, 0x2b, 0x77}, {0xc06c03e3c0, 0x2b, 0x55}, 0x1, 0xec) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0a95eeb88, {0xc0d84f8600, 0x1, 0x55}, {0xc06c03e3c0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0a95eeb88, {0xc0d84f8600, 0x2b, 0x77}, {0xc06c03e3c0, 0x2b, 0x55}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0a95ee2c0, {0xc06c03e3c0, 0x55, 0x77}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0a95ee2c0, {0xc0d84f8600, 0x2b, 0xc06c03e3c0}, {0xc06c03e3c0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0a95ee000, {0xc06c03e3c0, 0x2b, 0x55}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06daa5088, {{0xc01265dbc0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc060570e00, {{0xc01265dbc0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0529ab400, {0x63c85b8, 0xc073172db0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc073172db0}, {0x64a6560, 0xc060570e00}, {0xc01265dbc0, 0x4d545, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc073172db0}, {0x6466ed8, 0xc06daa5080}, {0xc01265dbc0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc026662d80}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0467b1a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020037000, {0x63c85b8, 0xc073172db0}, 0xc0467b1a00, 0xc0219d0fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020037000, {0x63c85b8, 0xc073172db0}, 0xc0467b1a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073172c90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073172c90}, {0xc0936948f0, 0x660089}, 0xc0936948f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073172c90}, {0xc00102e4e0}, 0xc0467b1900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073172c30}, 0xc0467b1900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0731728a0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811231 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02baf6b40, {0x63c85b8, 0xc0461a58f0}, 0xc02f8eacf0, 0x1, 0x0, 0x1, 0xc012e15a40, 0xc06c74e180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0461a58f0}, 0x120, 0xc0281aa728, 0x12e159e0, 0x0, 0x660069, 0xc012e15a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02baf6b40, {0x63c85b8, 0xc0461a58f0}, 0xc012e159e0, {{{{0x0, 0x0}, {0xc0b646f7a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0461a58f0}, 0x6352410, 0x6847e0e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bae0870, {0x63c85b8, 0xc0461a58f0}, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bae0870, {0x63c85b8, 0xc0461a58f0}, 0xc04544cb40, 0xc07c880ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0461a58f0}, 0x660089, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bb85400, {0x63c85b8, 0xc0461a58f0}, 0xc06b6e2e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0461a5860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0461a5860}, {0xc0109508f0, 0x660089}, 0xc0109508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0461a5860}, {0xc00102e4e0}, 0xc06b6e2c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0461a5800}, 0xc06b6e2c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc044fe7410}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801406 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0650d7aa0}, 0xc04da01cb0, {0x646cc48, 0xc0650d7b00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0650d7aa0}, 0xedb7a834f, 0xc036a785a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0650d7aa0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06ea88a20}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc04da01ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc06ea88a20}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812549 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021a5f540, {0x63c85b8, 0xc0566142a0}, 0xc0381616c8, 0x1, 0xc0416ff0a8, 0x519bbc, 0xc03e970780, 0xc008399620) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc03d292fc0, {0x63c85b8, 0xc0566142a0}, 0xc03e970780, 0xc0416ff1a8, 0x30, 0x40, 0x0, 0xc03e970780, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021a5f540, {0x63c85b8, 0xc0566142a0}, 0xc03e9706c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc05abc0a20, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0566142a0}, 0x6352260, 0x204d3270) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a30e60, {0x63c85b8, 0xc0566142a0}, {0xc03e9705a0, {0x174301104efe0834, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a30e60, {0x63c85b8, 0xc0566142a0}, 0xc02e743680, 0xc0416ff9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0566142a0}, 0x660089, {0xc03e9705a0, {0x174301104efe0834, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b2800, {0x63c85b8, 0xc0566142a0}, 0xc083616f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104efe0834, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104efe0834, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0566141e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0566141e0}, {0xc0417013f8, 0x660089}, 0xc041701400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0566141e0}, {0x7952446365435437}, 0xc083616d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056614150}, 0xc083616d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}, 0xc0ccf7fb00, 0xc038161b78) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}, 0x3, 0xc071e54800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc041701980}}}, {0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}, 0xc0ccf7fb00, 0xc071e54800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056614150}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056614150}, {0x40, 0xc041701b70}, 0xc041701ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056614150}, {0x4e9d7a0, 0xc083616d00}, 0xc0ccf7fb00, 0xc038161b78) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056614150}, 0xc05c2fbf80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea26c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea26c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811494 [runnable]: sync.runtime_SemacquireMutex(0xc034af1198, 0x8f, 0x7fd7365c9440) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc034af1040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x21, 0x18, 0x21, 0xc034af1068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc006e40d80, {0xffffffffffffff15, 0xc034af11c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc006e40d80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc006e40d80, {0xc025dcab00, 0x1f, 0x77}, {0xc0ab919a80, 0x1f, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0878d8398, {0xc025dcab00, 0x1f, 0x77}, {0xc0ab919a80, 0x1f, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0878d6cc0, {0xc0ab919a80, 0x74, 0x77}, 0x1f, 0x1f) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0878d6cc0, {0xc025dcab00, 0x74, 0xc0ab919a80}, {0xc0ab919a80, 0xc04b802b40, 0x1e}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0878d6a00, {0xc0ab919a80, 0x1f, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc052d38da8, {{0xc04b802b40, 0x1e, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc055bf7900, {{0xc0ae9e5968, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc061174400, {0x63c85b8, 0xc057b25fb0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc057b25fb0}, {0x64a6560, 0xc055bf7900}, {0xc0ae9e5968, 0x0, 0x18}, {0x1743011077a34578, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc057b25fb0}, {0x6466ed8, 0xc052d38b00}, {0xc0ae9e5968, 0x17, 0x18}, {0x1743011077a34578, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc027d299c8}, {{0x1743011077a34578, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011077a34578, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05afaca00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174301109570aa78, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02fc85c00, {0x63c85b8, 0xc057b25fb0}, 0xc05afaca00, 0xc0536b7220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fc85c00, {0x63c85b8, 0xc057b25fb0}, 0xc05afaca00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077a34578, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077a34578, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057b25e90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057b25e90}, {0xc034af73f8, 0x660089}, 0xc034af7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057b25e90}, {0x7fe42864db30}, 0xc05afac800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057b25d40}, 0xc05afac800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}, 0xc020cd52c0, 0xc016ee7a40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}, 0x3, 0xc03d1eb1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034af7980}}}, {0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}, 0xc020cd52c0, 0xc03d1eb1c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc057b25d40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057b25d40}, {0x40, 0xc034af7b70}, 0xc034af7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc057b25d40}, {0x4e9d7a0, 0xc05afac800}, 0xc020cd52c0, 0xc016ee7a40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc057b25d40}, 0xc04b394420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc037c1c240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc037c1c240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804579 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc031d665d0}, 0xc053fc4c60, 0x1, 0xc077ae56b8, 0x519bbc, 0xc05bb4e7e0, 0xc04d339aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05efac840, {0x63c85b8, 0xc031d665d0}, 0xc089989680, 0xc077ae57b8, 0x5bb4e7e0, 0x0, 0x0, 0xc05bb4e7e0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc031d665d0}, 0xc05bb4e780, {{{{0xc06118ca20, 0x14}, {0xc01d2478c0, 0x20}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc031d665d0}, 0x63525f0, 0x41bc5e90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc031d665d0}, {0x0, {0x17430110355442dd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc031d665d0}, 0xc02b0a39a0, 0xc077ae6000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc031d665d0}, 0x660089, {0x0, {0x17430110355442dd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc031d665d0}, 0xc06b8bcf00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110355442dd, 0x0, 0x0}, 0xc03de53170, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110355442dd, 0x0, 0x0}, 0xc03de53170, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031d66510}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031d66510}, {0xc07b0d5a08, 0x660089}, 0xc07b0d5a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031d66510}, {0xc00102e4e0}, 0xc06b8bce00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc031d664b0}, 0xc06b8bce00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc031d66420}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc031d66420}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219793081 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0336137a0}, 0xc012277cb0, {0x646cc48, 0xc033613860}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0336137a0}, 0xedb7a834e, 0xc095dd4a20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0336137a0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc044b76c00}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc012277ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc044b76c00}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812266 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc053ec7350}, 0xc04c38fb30, 0x1, 0x7080181bbd912a, 0x37e11d600, 0xc013d7b6e0, 0xc091397800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc053ec7350}, 0x120, 0xc037dab230, 0x13d7b680, 0x0, 0x660069, 0xc013d7b6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc053ec7350}, 0xc013d7b680, {{{{0x0, 0x0}, {0xc08f7a4900, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc053ec7350}, 0x6352410, 0x65f8a690) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc053ec7350}, {0xc013d7b560, {0x1743011077a7adfe, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc053ec7350}, 0xc071187720, 0xc037dab9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc053ec7350}, 0x660089, {0xc013d7b560, {0x1743011077a7adfe, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc053ec7350}, 0xc04ecf6400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077a7adfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077a7adfe, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053ec72c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053ec72c0}, {0xc037dad3f8, 0x660089}, 0xc037dad400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053ec72c0}, {0x7fe3263296d0}, 0xc04ecf6200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053ec7230}, 0xc04ecf6200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}, 0xc03294fee0, 0xc04c38fb00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}, 0x3, 0xc019e1e000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc037dad980}}}, {0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}, 0xc03294fee0, 0xc019e1e000) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc053ec7230}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053ec7230}, {0x40, 0xc037dadb70}, 0xc037dadab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc053ec7230}, {0x4e9d7a0, 0xc04ecf6200}, 0xc03294fee0, 0xc04c38fb00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc053ec7230}, 0xc0653351a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc013d7b440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc013d7b440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804803 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc082aa90e0}, 0xc0420ac798, 0x1, 0xc0958e50a8, 0x519bbc, 0xc06b794cc0, 0xc045944a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc082aa90e0}, 0x120, 0xc0958e5230, 0x6b794c60, 0xc0d08c7040, 0x18, 0xc06b794cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc082aa90e0}, 0xc06b794c60, {{{{0x0, 0x0}, {0xc0547b4240, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc082aa90e0}, 0x6352410, 0x199f0460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc082aa90e0}, {0xc06b794b40, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc082aa90e0}, 0xc0686114a0, 0xc0958e59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc082aa90e0}, 0x660089, {0xc06b794b40, {0x174301101d4da0e3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc082aa90e0}, 0xc0ae80c400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d4da0e3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc082aa9050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082aa9050}, {0xc0958e73f8, 0x660089}, 0xc0958e7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc082aa9050}, {0x7fe2ff563dd8}, 0xc0ae80c200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc082aa8f90}, 0xc0ae80c200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}, 0xc023048d80, 0xc0420ad650) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}, 0x3, 0xc00ecc67c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0958e7980}}}, {0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}, 0xc023048d80, 0xc00ecc67c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc082aa8f90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc082aa8f90}, {0x40, 0xc0958e7b70}, 0xc0958e7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}, 0x203003, 0x203003) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc082aa8f90}, {0x4e9d7a0, 0xc0ae80c200}, 0xc023048d80, 0xc0420ad650) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc082aa8f90}, 0xc06c7379e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af93560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc06af93560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812611 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x80, 0xc01fce02c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x4a089a9cc0400800, 0xc052acc380, 0xc041839168) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc01fce02c0, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc052acc380, {0xc06342fc20, 0x28, 0x28}, {0xc059cc1860, 0x28, 0x57}, 0x0, 0x48) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0925546e0, {0xc06342fc20, 0x1, 0x57}, {0xc059cc1860, 0xc092553900, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0925546e0, {0xc06342fc20, 0x28, 0x28}, {0xc059cc1860, 0x28, 0x57}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092553bc0, {0xc059cc1860, 0x4a2372, 0x28}, 0x464ed60, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092553bc0, {0xc06342fc20, 0x28, 0x57}, {0xc059cc1860, 0x27, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092553900, {0xc059cc1860, 0x28, 0x57}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc062f96688, {{0xc06342fbc0, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc050c03700, {{0xc06342fbc0, 0x27, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ff98c00, {0x63c85b8, 0xc04f63b9e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04f63b9e0}, {0x64a6560, 0xc050c03700}, {0xc06342fbc0, 0x80a2a8801003d3d, 0x30}, {0x1743011094fed0c7, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04f63b9e0}, {0x6466ed8, 0xc062f96680}, {0xc06342fbc0, 0x27, 0x30}, {0x1743011094fed0c7, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc037113788}, {{0x1743011094fed0c7, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011094fed0c7, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0455a3a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b2cc35c7, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02f355c00, {0x63c85b8, 0xc04f63b9e0}, 0xc0455a3a00, 0xc051f75a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02f355c00, {0x63c85b8, 0xc04f63b9e0}, 0xc0455a3a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011094fed0c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011094fed0c7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f63b920}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b920}, {0xc04183f3f8, 0x660089}, 0xc04183f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f63b920}, {0x6fab021001431700}, 0xc0455a3700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f63b890}, 0xc0455a3700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}, 0xc03ebdd5e0, 0xc031cfb158) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}, 0x3, 0xc033d33bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04183f980}}}, {0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}, 0xc03ebdd5e0, 0xc033d33bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f63b890}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f63b890}, {0x40, 0xc04183fb70}, 0xc04183fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f63b890}, {0x4e9d7a0, 0xc0455a3700}, 0xc03ebdd5e0, 0xc031cfb158) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f63b890}, 0xc039f0aae0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e065e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03e065e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800866 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bd5e000, {0x63c85b8, 0xc07bda0390}, 0xc015754d00, 0xc04d8a6780) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bd5e000, {0x63c85b8, 0xc07bda0390}, 0xc015754d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110165bb3d0, 0x0, 0x0}, 0xc01a37af80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110165bb3d0, 0x0, 0x0}, 0xc01a37af80, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07bda01b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bda01b0}, {0xc03eeab3f8, 0x660089}, 0xc03eeab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07bda01b0}, {0x7fe429ea8340}, 0xc045f31f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0213edfb0}, 0xc045f31f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}, 0xc01f0f9e60, 0xc037048ae0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}, 0x3, 0xc07632ba80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03eeab980}}}, {0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}, 0xc01f0f9e60, 0xc07632ba80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0213edfb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0213edfb0}, {0x40, 0xc03eeabb70}, 0xc03eeabab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0213edfb0}, {0x4e9d7a0, 0xc045f31f00}, 0xc01f0f9e60, 0xc037048ae0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0213edfb0}, 0xc03a41b9e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05c253680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05c253680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812636 [runnable]: internal/poll.runtime_pollWait(0x7fdbafe79a98, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06481a700, 0xc052e23b00, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06481a700, {0xc052e23b00, 0x205, 0x205}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06481a700, {0xc052e23b00, 0x4d91a5, 0x9f63bb62b2ea5ead}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc033a92e30, {0xc052e23b00, 0x4d858f, 0xc04e5e9f90}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04d73ade0, {0xc052e23b00, 0x65, 0x205}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc060f8e6d8, {0xc052e23b00, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc03b6333f8, {0x631b900, 0xc060f8e6d8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc03b633180, {0x7fe42ab59310, 0xc04d73ade0}, 0x12835b015807aa98) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc03b633180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).readHandshake(0xc03b633180) GOROOT/src/crypto/tls/conn.go:1010 +0x6d crypto/tls.(*serverHandshakeStateTLS13).readClientCertificate(0xc088139148) GOROOT/src/crypto/tls/handshake_server_tls13.go:783 +0x65 crypto/tls.(*serverHandshakeStateTLS13).handshake(0xc088139148) GOROOT/src/crypto/tls/handshake_server_tls13.go:74 +0xb6 crypto/tls.(*Conn).serverHandshake(0xc03b633180, {0x63c8510, 0xc050a070c0}) GOROOT/src/crypto/tls/handshake_server.go:54 +0xd0 crypto/tls.(*Conn).handshakeContext(0xc03b633180, {0x63c8548, 0xc000140068}) GOROOT/src/crypto/tls/conn.go:1453 +0x36f crypto/tls.(*Conn).HandshakeContext(...) GOROOT/src/crypto/tls/conn.go:1396 crypto/tls.(*Conn).Handshake(...) GOROOT/src/crypto/tls/conn.go:1380 crypto/tls.(*Conn).Read(0xc03b633180, {0xc073d14498, 0x4, 0xc019549520}) GOROOT/src/crypto/tls/conn.go:1265 +0x65 io.ReadAtLeast({0x7fe456a34450, 0xc03b633180}, {0xc073d14498, 0x4, 0x4}, 0x4) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadUntypedMsg(0xc073d14480, {0x7fe456a34450, 0xc03b633180}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:151 +0x65 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).readVersion(0xc00c689e00, {0x7fe456a34450, 0xc03b633180}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1357 +0x11f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).maybeUpgradeToSecureConn(0xc00c689e00, {0x0, 0xffc}, {0x643d810, 0xc04d73ade0}, 0x2, 0x4d2162f, 0xc088139be0) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1301 +0x367 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c8510, 0xc050a07080}, {0x643d810, 0xc04d73ade0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:807 +0x5a9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc073d143c0}, {0x643d810, 0xc04d73ade0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc073d143c0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808259 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc06f432240}, 0xc05fe67608, 0x1, 0x0, 0x1, 0xc07a423c80, 0xc0741e8840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06f432240}, 0x120, 0xc0059de728, 0x7a423c20, 0x0, 0x660069, 0xc07a423c80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc06f432240}, 0xc07a423c20, {{{{0x0, 0x0}, {0xc0735de000, 0x2a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06f432240}, 0x6352410, 0x2ad6f410) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc06f432240}, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc06f432240}, 0xc06477dae0, 0xc082d14ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f432240}, 0x660089, {0xc07cd10360, {0x1743011050664d97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc06f432240}, 0xc084bb4700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050664d97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f4320f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f4320f0}, {0xc00310a8f0, 0x660089}, 0xc00310a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f4320f0}, {0xc00102e4e0}, 0xc084bb4600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f432090}, 0xc084bb4600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc045e73620}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812409 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc04f6fccc0}, 0xc03fc3d8f0, 0x1, 0xc03f2cf0a8, 0x519bbc, 0xc05a688f60, 0xc01ada29c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04f6fccc0}, 0x1e0, 0xc03f2cf230, 0x5a688f00, 0x0, 0x660069, 0xc05a688f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc04f6fccc0}, 0xc05a688f00, {{{{0x0, 0x0}, {0xc03081f200, 0x6d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04f6fccc0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc04f6fccc0}, {0xc05396d9e0, {0x174301107ca83f7d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc04f6fccc0}, 0xc0519988c0, 0xc03f2cf9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f6fccc0}, 0x660089, {0xc05396d9e0, {0x174301107ca83f7d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc04f6fccc0}, 0xc079621e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107ca83f7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107ca83f7d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f6fcc30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f6fcc30}, {0xc03f2d13f8, 0x660089}, 0xc03f2d1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f6fcc30}, {0x7fe429e510d0}, 0xc079621b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f6fcba0}, 0xc079621b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}, 0xc087886780, 0xc043a91da0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}, 0x3, 0xc058616ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03f2d1980}}}, {0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}, 0xc087886780, 0xc058616ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f6fcba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f6fcba0}, {0x40, 0xc03f2d1b70}, 0xc03f2d1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f6fcba0}, {0x4e9d7a0, 0xc079621b00}, 0xc087886780, 0xc043a91da0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f6fcba0}, 0xc04d5f7c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05396cc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05396cc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812014 [runnable]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc073d22cc0}, 0xc054532ac8, 0x1, 0xc02f1e90a8, 0x519bbc, 0xc059ec9140, 0xc05a2dcba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc073d22cc0}, 0x1e0, 0xc02f1e9230, 0x59ec90e0, 0x0, 0x660069, 0xc059ec9140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc073d22cc0}, 0xc059ec90e0, {{{{0x0, 0x0}, {0xc04e12c000, 0x11f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073d22cc0}, 0x63521a0, 0x3abf6528) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc073d22cc0}, {0xc01819ec60, {0x174301107dee08fa, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc073d22cc0}, 0xc0689760a0, 0xc02f1e99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073d22cc0}, 0x660089, {0xc01819ec60, {0x174301107dee08fa, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc073d22cc0}, 0xc065028500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107dee08fa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107dee08fa, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073d22c30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d22c30}, {0xc02f1eb3f8, 0x660089}, 0xc02f1eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073d22c30}, {0x174300f4a071d600}, 0xc065028300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073d22ba0}, 0xc065028300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}, 0xc0486a66c0, 0xc054532eb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}, 0x3, 0xc060f9a300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02f1eb980}}}, {0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}, 0xc0486a66c0, 0xc060f9a300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073d22ba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073d22ba0}, {0x40, 0xc02f1ebb70}, 0xc02f1ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073d22ba0}, {0x4e9d7a0, 0xc065028300}, 0xc0486a66c0, 0xc054532eb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073d22ba0}, 0xc044fe5aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05153c5a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05153c5a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804599 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc06d1917a0}, 0xc027447710, 0x1, 0xc078e2d0a8, 0x519bbc, 0xc053184060, 0xc051d40060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc06d1917a0}, 0x1e0, 0xc078e2d230, 0x53184000, 0x0, 0x660069, 0xc053184060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc06d1917a0}, 0xc053184000, {{{{0x0, 0x0}, {0xc09cba9440, 0xf2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x13000000000000, {0x63c85b8, 0xc06d1917a0}, 0xa000120000, 0x50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc06d1917a0}, {0xc066b8d320, {0x174301101d654fc6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc06d1917a0}, 0xc0689761e0, 0xc078e2d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06d1917a0}, 0x660089, {0xc066b8d320, {0x174301101d654fc6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc06d1917a0}, 0xc0701b9500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d654fc6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d654fc6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06d191710}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d191710}, {0xc078e2f3f8, 0x660089}, 0xc078e2f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06d191710}, {0x0}, 0xc0701b9300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06d191680}, 0xc0701b9300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}, 0xc074629dc0, 0xc050158618) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}, 0x3, 0xc05ec88300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078e2f980}}}, {0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}, 0xc074629dc0, 0xc05ec88300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06d191680}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d191680}, {0x40, 0xc078e2fb70}, 0xc078e2fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06d191680}, {0x4e9d7a0, 0xc0701b9300}, 0xc074629dc0, 0xc050158618) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06d191680}, 0xc0442498c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0671b1e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0671b1e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809581 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc000c98800, {0x63c85b8, 0xc057b1fe30}, 0xc05af2f200, 0xc0225cfa40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000c98800, {0x63c85b8, 0xc057b1fe30}, 0xc05af2f200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067e6af4a, 0x0, 0x0}, 0xc016090080, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067e6af4a, 0x0, 0x0}, 0xc016090080, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x10, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc057b1fda0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057b1fda0}, {0xc0064eb3f8, 0x660089}, 0xc0064eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc057b1fda0}, {0x3ecf270a522403e2}, 0xc05af2ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc057b1fd10}, 0xc05af2ef00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}, 0xc0491fec40, 0xc04774c6c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}, 0x3, 0xc017ac4c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0064eb980}}}, {0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}, 0xc0491fec40, 0xc017ac4c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc057b1fd10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc057b1fd10}, {0x40, 0xc0064ebb70}, 0xc0064ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc057b1fd10}, {0x4e9d7a0, 0xc05af2ef00}, 0xc0491fec40, 0xc04774c6c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc057b1fd10}, 0xc022f15080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0843d79e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0843d79e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810789 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc07995a4b0}, 0xc06292ec60, 0x1, 0xc0264790a8, 0x519bbc, 0xc04a5b7ec0, 0xc051d92ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07995a4b0}, 0x120, 0xc026479230, 0x4a5b7e60, 0x0, 0x660069, 0xc04a5b7ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc07995a4b0}, 0xc04a5b7e60, {{{{0x0, 0x0}, {0xc05377dc20, 0xb}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07995a4b0}, 0x6352410, 0x55bf2468) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc07995a4b0}, {0xc04a5b7d40, {0x1743011063852431, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc07995a4b0}, 0xc021d49360, 0xc0264799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07995a4b0}, 0x660089, {0xc04a5b7d40, {0x1743011063852431, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc07995a4b0}, 0xc086e92e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063852431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063852431, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07995a3f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07995a3f0}, {0xc02647b3f8, 0x660089}, 0xc02647b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07995a3f0}, {0x4e4c674b77684a75}, 0xc086e92c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07995a360}, 0xc086e92c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}, 0xc06734c380, 0xc01c4380d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}, 0x3, 0xc015d26c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02647b980}}}, {0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}, 0xc06734c380, 0xc015d26c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07995a360}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07995a360}, {0x40, 0xc02647bb70}, 0xc02647bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07995a360}, {0x4e9d7a0, 0xc086e92c00}, 0xc06734c380, 0xc01c4380d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07995a360}, 0xc03090fce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04a5b7c20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04a5b7c20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810607 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0c73de9c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804880 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc048382240}, 0xc031857c38, 0x1, 0xc097a4b0a8, 0x519bbc, 0xc06f85cba0, 0xc04d1dd140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc048382240}, 0x120, 0xc097a4b230, 0x6f85cb40, 0xc0cdc431e0, 0x18, 0xc06f85cba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc048382240}, 0xc06f85cb40, {{{{0x0, 0x0}, {0xc0477b4ea0, 0x19}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc048382240}, 0x6352410, 0x1b25d2c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc048382240}, {0xc06f85ca20, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc048382240}, 0xc02a82fd60, 0xc097a4b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc048382240}, 0x660089, {0xc06f85ca20, {0x174301101d1bdf29, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc048382240}, 0xc0878b1a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101d1bdf29, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0483821b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0483821b0}, {0xc097a4d3f8, 0x660089}, 0xc097a4d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0483821b0}, {0x56796b6c64346330}, 0xc0878b1800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc048382120}, 0xc0878b1800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}, 0xc0235c5240, 0xc0483f6e58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}, 0x3, 0xc01ae265c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc097a4d980}}}, {0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}, 0xc0235c5240, 0xc01ae265c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc048382120}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc048382120}, {0x40, 0xc097a4db70}, 0xc097a4dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}, 0x203006, 0x203006) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc048382120}, {0x4e9d7a0, 0xc0878b1800}, 0xc0235c5240, 0xc0483f6e58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc048382120}, 0xc02e675aa0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85c900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06f85c900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802987 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01df528c0, {0x63c85b8, 0xc03b7961b0}, 0xc035b7bf20, 0x1, 0xc06b8530a8, 0x519bbc, 0xc0585f4720, 0xc0ba6443c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03b7961b0}, 0x120, 0xc06b853230, 0x585f46c0, 0x0, 0x660069, 0xc0585f4720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01df528c0, {0x63c85b8, 0xc03b7961b0}, 0xc0585f46c0, {{{{0x0, 0x0}, {0xc05c772360, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03b7961b0}, 0x6352410, 0x3080e448) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dec4f00, {0x63c85b8, 0xc03b7961b0}, {0xc0585f45a0, {0x1743011027f78ab5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dec4f00, {0x63c85b8, 0xc03b7961b0}, 0xc079989f40, 0xc06b8539f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03b7961b0}, 0x660089, {0xc0585f45a0, {0x1743011027f78ab5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01df61400, {0x63c85b8, 0xc03b7961b0}, 0xc0aa4ac700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011027f78ab5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03b796120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b796120}, {0xc06b8553f8, 0x660089}, 0xc06b855400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03b796120}, {0x7fe3e2122a70}, 0xc0aa4ac500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03b796090}, 0xc0aa4ac500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}, 0xc008405100, 0xc01a542f30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}, 0x3, 0xc0399ff100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06b855980}}}, {0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}, 0xc008405100, 0xc0399ff100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03b796090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b796090}, {0x40, 0xc06b855b70}, 0xc06b855ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03b796090}, {0x4e9d7a0, 0xc0aa4ac500}, 0xc008405100, 0xc01a542f30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03b796090}, 0xc028e7ea80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075f55680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075f55680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812800 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809445 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc077a15ad0}, 0xc0614fd1d0, 0x1, 0x0, 0x1, 0xc0b11a2f00, 0xc06f310ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077a15ad0}, 0x120, 0xc0268e4728, 0xb11a2ea0, 0x0, 0x660069, 0xc0b11a2f00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc077a15ad0}, 0xc0b11a2ea0, {{{{0x0, 0x0}, {0xc057369c20, 0xc}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077a15ad0}, 0x6352410, 0x479968f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc077a15ad0}, {0xc0d27d2c60, {0x17430110641b5d51, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc077a15ad0}, 0xc02ad1e6e0, 0xc070134ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077a15ad0}, 0x660089, {0xc0d27d2c60, {0x17430110641b5d51, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc077a15ad0}, 0xc00c027d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110641b5d51, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110641b5d51, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077a15a40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077a15a40}, {0xc002fea8f0, 0x660089}, 0xc002fea8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077a15a40}, {0xc00102e4e0}, 0xc00c027c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc077a159e0}, 0xc00c027c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02aebc930}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812647 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800854 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc083fe2548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc083fe25b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc091c7c5f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0d0f7a700, {0xc00f80c850, 0x2b, 0x6c}, {0xc06432f500, 0x2b, 0x6e}, 0x1, 0xc5) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091c7c488, {0xc00f80c850, 0x1, 0x6e}, {0xc06432f500, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091c7c488, {0xc00f80c850, 0x2b, 0x6c}, {0xc06432f500, 0x2b, 0x6e}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091c7bbc0, {0xc06432f500, 0x6e, 0x6c}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091c7bbc0, {0xc00f80c850, 0x2b, 0xc06432f500}, {0xc06432f500, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091c7b900, {0xc06432f500, 0x2b, 0x6e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04eda9b88, {{0xc048ad6ae0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc060ccbe00, {{0xc048ad6ae0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc05f7b6c00, {0x63c85b8, 0xc04b741da0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04b741da0}, {0x64a6560, 0xc060ccbe00}, {0xc048ad6ae0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04b741da0}, {0x6466ed8, 0xc04eda9b80}, {0xc048ad6ae0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc026662a80}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0401eaf00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c00c00, {0x63c85b8, 0xc04b741da0}, 0xc0401eaf00, 0xc066746b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c00c00, {0x63c85b8, 0xc04b741da0}, 0xc0401eaf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04b741d10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04b741d10}, {0xc0032fe8f0, 0x660089}, 0xc0032fe8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04b741d10}, {0xc00102e4e0}, 0xc0401eae00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04b741cb0}, 0xc0401eae00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06d9a4600}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801129 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc026c70720}, 0xc032b1dcb0, {0x646cc48, 0xc026c707e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc026c70720}, 0xedb7a834e, 0xc00df60480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc026c70720}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03a24c720}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc032b1def8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc03a24c720}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809048 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020b437c0, {0x63c85b8, 0xc04f26f410}, 0xc0672e9b00, 0x1, 0xc021e490a8, 0x519bbc, 0xc06be40a80, 0xc0648c35c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04f26f410}, 0x120, 0xc021e49230, 0x6be40a20, 0x0, 0x660069, 0xc06be40a80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020b437c0, {0x63c85b8, 0xc04f26f410}, 0xc06be40a20, {{{{0x0, 0x0}, {0xc051479440, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f26f410}, 0x6352410, 0x450caa98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc020bfe910, {0x63c85b8, 0xc04f26f410}, {0xc06be40900, {0x174301105e6e348b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc020bfe910, {0x63c85b8, 0xc04f26f410}, 0xc052bbadc0, 0xc021e499f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f26f410}, 0x660089, {0xc06be40900, {0x174301105e6e348b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ed79400, {0x63c85b8, 0xc04f26f410}, 0xc0360b5a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e6e348b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e6e348b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f26f350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f26f350}, {0xc021e4b3f8, 0x660089}, 0xc021e4b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f26f350}, {0x0}, 0xc0360b5800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f26f2c0}, 0xc0360b5800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}, 0xc0707cee60, 0xc05eae14d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}, 0x3, 0xc06890b840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc021e4b980}}}, {0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}, 0xc0707cee60, 0xc06890b840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f26f2c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f26f2c0}, {0x40, 0xc021e4bb70}, 0xc021e4bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f26f2c0}, {0x4e9d7a0, 0xc0360b5800}, 0xc0707cee60, 0xc05eae14d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f26f2c0}, 0xc054015380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06be407e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06be407e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802282 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc06714a030}, 0xc03c8bf248, 0x1, 0xc05f7e90a8, 0x519bbc, 0xc021351020, 0xc06794d800) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06714a030}, 0x120, 0xc05f7e9230, 0x21350fc0, 0x0, 0x660069, 0xc021351020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc06714a030}, 0xc021350fc0, {{{{0x0, 0x0}, {0xc02957e480, 0xb6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06714a030}, 0x6352410, 0x1b10ddc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc06714a030}, {0xc021350ea0, {0x1743011020e2c87a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc06714a030}, 0xc078204500, 0xc05f7e99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06714a030}, 0x660089, {0xc021350ea0, {0x1743011020e2c87a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc06714a030}, 0xc04e8caa00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020e2c87a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020e2c87a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066d31e00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066d31e00}, {0xc05f7eb3f8, 0x660089}, 0xc05f7eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066d31e00}, {0xc0015221e0}, 0xc04e8ca700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066d31b00}, 0xc04e8ca700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}, 0xc08a6d1860, 0xc0364c18c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}, 0x3, 0xc03f5e8540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05f7eb980}}}, {0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}, 0xc08a6d1860, 0xc03f5e8540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066d31b00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066d31b00}, {0x40, 0xc05f7ebb70}, 0xc05f7ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066d31b00}, {0x4e9d7a0, 0xc04e8ca700}, 0xc08a6d1860, 0xc0364c18c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066d31b00}, 0xc028bee120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc014e36480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc014e36480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807398 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a63c0, {0x63c85b8, 0xc04b5a9770}, 0xc01cb7eb58, 0x1, 0xc0033e70a8, 0x519bbc, 0xc0b60073c0, 0xc06fbb48a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01b225c20, {0x63c85b8, 0xc04b5a9770}, 0xc0b60073c0, 0xc0033e71a8, 0x41, 0x60, 0x2, 0xc0b60073c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a63c0, {0x63c85b8, 0xc04b5a9770}, 0xc0b6007180, {{{{0x0, 0x0}, {0xc06f376fc0, 0x3}}, {{0xc02118ca20, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04b5a9770}, 0x63521a0, 0x23d19328) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddd10, {0x63c85b8, 0xc04b5a9770}, {0xc081fd4900, {0x1743011000b8173b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddd10, {0x63c85b8, 0xc04b5a9770}, 0xc068881a40, 0xc0033e79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04b5a9770}, 0x660089, {0xc081fd4900, {0x1743011000b8173b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e09b400, {0x63c85b8, 0xc04b5a9770}, 0xc092036000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011000b8173b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011000b8173b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04b5a96e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04b5a96e0}, {0xc0033e93f8, 0x660089}, 0xc0033e9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04b5a96e0}, {0x0}, 0xc08b8c5e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04b5a9590}, 0xc08b8c5e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}, 0xc007b4e4c0, 0xc0469710b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}, 0x3, 0xc039e7a880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0033e9980}}}, {0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}, 0xc007b4e4c0, 0xc039e7a880) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04b5a9590}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04b5a9590}, {0x40, 0xc0033e9b70}, 0xc0033e9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04b5a9590}, {0x4e9d7a0, 0xc08b8c5e00}, 0xc007b4e4c0, 0xc0469710b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04b5a9590}, 0xc05205ed20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081fd47e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc081fd47e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810381 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc02762ad20}, 0xc03860c5d0, 0x1, 0xc028cf10a8, 0x519bbc, 0xc0722c33e0, 0xc06404e0c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06cc875c0, {0x63c85b8, 0xc02762ad20}, 0xc089989440, 0xc028cf11a8, 0x722c33e0, 0x0, 0x2, 0xc0722c33e0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc02762ad20}, 0xc0722c3320, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc01c5d2b40, 0xa4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02762ad20}, 0x6352260, 0x51b88e10) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc02762ad20}, {0xc0722c3200, {0x1743011031e4a909, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02762ad20}, 0xc06c7bd5e0, 0xc028cf19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02762ad20}, 0x660089, {0xc0722c3200, {0x1743011031e4a909, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02762ad20}, 0xc059279200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011031e4a909, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02762ac90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02762ac90}, {0xc028cf33f8, 0x660089}, 0xc028cf3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02762ac90}, {0xc0015221e0}, 0xc059279000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02762ab70}, 0xc059279000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}, 0xc07d80c4e0, 0xc02366b908) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}, 0x3, 0xc07a00b640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028cf3980}}}, {0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}, 0xc07d80c4e0, 0xc07a00b640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02762ab70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02762ab70}, {0x40, 0xc028cf3b70}, 0xc028cf3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02762ab70}, {0x4e9d7a0, 0xc059279000}, 0xc07d80c4e0, 0xc02366b908) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02762ab70}, 0xc06bf02f00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0722c30e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0722c30e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810758 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0656ed8f0}, 0xc03c10d248, 0x1, 0xc0209930a8, 0x519bbc, 0xc01e77dec0, 0xc091a5a3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc055972240, {0x63c85b8, 0xc0656ed8f0}, 0xc0380a7320, 0xc0209931a8, 0x1c97ccf, 0xc036cb6900, 0x2, 0xc01e77dec0, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0656ed8f0}, 0xc01e77de60, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc045559d40, 0xa8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc0656ed8f0}, 0x63523b0, 0x30992710) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0656ed8f0}, {0x0, {0x174301106f75b4c0, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0656ed8f0}, 0xc07c1fa000, 0xc0209939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0656ed8f0}, 0x660089, {0x0, {0x174301106f75b4c0, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0656ed8f0}, 0xc088ab9a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106f75b4c0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106f75b4c0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301106f75b4c0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0656ed860}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0656ed860}, {0xc0209953f8, 0x660089}, 0xc020995400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0656ed860}, {0x7fe4292605d0}, 0xc088ab9800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0656ed7d0}, 0xc088ab9800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}, 0xc0505d83e0, 0xc0252ff3b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}, 0x3, 0xc0299a2400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc020995980}}}, {0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}, 0xc0505d83e0, 0xc0299a2400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0656ed7d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0656ed7d0}, {0x40, 0xc020995b70}, 0xc020995ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0656ed7d0}, {0x4e9d7a0, 0xc088ab9800}, 0xc0505d83e0, 0xc0252ff3b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0656ed7d0}, 0xc04190c900, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc01e77dd40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc01e77dd40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811805 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b9a2f00, {0x63c85b8, 0xc023b27aa0}, 0xc020b9c0d8, 0x1, 0xc0340350a8, 0x519bbc, 0xc02b507920, 0xc019fde720) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc023b27aa0}, 0x120, 0xc034035230, 0x2b5078c0, 0x0, 0x660069, 0xc02b507920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b9a2f00, {0x63c85b8, 0xc023b27aa0}, 0xc02b5078c0, {{{{0x0, 0x0}, {0xc044134a20, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc023b27aa0}, 0x6352410, 0x6b0663c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b0fbb30, {0x63c85b8, 0xc023b27aa0}, {0xc02b507440, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b0fbb30, {0x63c85b8, 0xc023b27aa0}, 0xc04d955360, 0xc0340359f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc023b27aa0}, 0x660089, {0xc02b507440, {0x1743011079222622, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b9bd400, {0x63c85b8, 0xc023b27aa0}, 0xc02029d400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011079222622, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023b27a10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023b27a10}, {0xc0340373f8, 0x660089}, 0xc034037400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023b27a10}, {0x7fe3262b6a78}, 0xc02029d200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc023b27980}, 0xc02029d200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}, 0xc08d07aa20, 0xc021240378) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}, 0x3, 0xc01586dd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034037980}}}, {0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}, 0xc08d07aa20, 0xc01586dd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc023b27980}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023b27980}, {0x40, 0xc034037b70}, 0xc034037ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc023b27980}, {0x4e9d7a0, 0xc02029d200}, 0xc08d07aa20, 0xc021240378) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc023b27980}, 0xc065ea56e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04565c360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04565c360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219797931 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc025138e40}, 0xc018834cb0, {0x646cc48, 0xc025138ea0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc025138e40}, 0xedb7a834e, 0xc00ba30b40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc025138e40}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04fa30510}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc018834ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc04fa30510}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810353 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc084487f80}, 0xc0659e48a0, 0x1, 0xc0293a10a8, 0x519bbc, 0xc05c363b60, 0xc012755020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc084487f80}, 0x120, 0xc0293a1230, 0x5c363b00, 0x0, 0x660069, 0xc05c363b60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc084487f80}, 0xc05c363b00, {{{{0x0, 0x0}, {0xc06b650b40, 0x22}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc084487f80}, 0x6352410, 0x30992388) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc084487f80}, {0xc05c3639e0, {0x174301106d6d1d47, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc084487f80}, 0xc02c449b80, 0xc0293a19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc084487f80}, 0x660089, {0xc05c3639e0, {0x174301106d6d1d47, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc084487f80}, 0xc051e1b800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d6d1d47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d6d1d47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc084487ef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc084487ef0}, {0xc0293a33f8, 0x660089}, 0xc0293a3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc084487ef0}, {0xc0015221e0}, 0xc051e1b600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc084487e60}, 0xc051e1b600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}, 0xc04802e800, 0xc0b58e7f50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}, 0x3, 0xc0210b4c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0293a3980}}}, {0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}, 0xc04802e800, 0xc0210b4c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc084487e60}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc084487e60}, {0x40, 0xc0293a3b70}, 0xc0293a3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc084487e60}, {0x4e9d7a0, 0xc051e1b600}, 0xc04802e800, 0xc0b58e7f50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc084487e60}, 0xc0755c5c20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0483c8c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0483c8c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810040 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc024915260}, 0xc06ba8e690, 0x1, 0xc004c7d670, 0x519bbc, 0xc07e1cf740, 0xc05f7fbaa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc05abf78c0, {0x63c85b8, 0xc024915260}, 0xc05abf7970, 0xc004c7d770, 0x27, 0x40, 0xffffffffffffffff, 0xc07e1cf740, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc024915260}, 0xc07e1cf500, {{{{0xc05fad0c60, 0x10}, {0xc0441350e0, 0x1e}}, {{0xc0408af8c0, 0x3e}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc024915260}, 0x63521a0, 0x43a28b58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc024915260}, {0xc04b38e000, {0x174301100997be8d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc024915260}, 0xc06da7fea0, 0xc004c7dfb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024915260}, 0x660089, {0xc04b38e000, {0x174301100997be8d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc024915260}, 0xc0855f5c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100997be8d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100997be8d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0249150b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0249150b0}, {0xc0766b79c0, 0x660089}, 0xc0766b79c8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0249150b0}, {0xc00102e4e0}, 0xc0855f5b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024915050}, 0xc0855f5b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x63c85b8, 0xc0249145d0}, {0x7fe422a51230, 0xc0ce411580}, 0xc01c97aea0, {0xc04d6023f0, 0x3, 0x3}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x24d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x63c85b8, 0xc0249145d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812732 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc084303c00, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc032993198, 0xc09475df38, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc09475e050, 0x0, 0xc09475e050) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc09475dee8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09475dee8, {0xc023ee9e30, 0x18, 0x6f}, {0xc08b77ec00, 0x18, 0x28}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09475ccc0, {0xc08b77ec00, 0x0, 0xc0329934e0}, 0x121ea30, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09475ccc0, {0xc023ee9e30, 0x18, 0x28}, {0xc08b77ec00, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09475ca00, {0xc08b77ec00, 0x18, 0x28}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc0724ac008, {{0xc08e777500, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc08dbb3900, {{0xc08e777500, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04b393000, {0x63c85b8, 0xc062cdf3e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062cdf3e0}, {0x64a6560, 0xc08dbb3900}, {0xc08e777500, 0xc03fcbc1a0, 0x18}, {0x17430110981f2118, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062cdf3e0}, {0x6466ed8, 0xc0724ac000}, {0xc08e777500, 0x17, 0x18}, {0x17430110981f2118, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c5ec558}, {{0x17430110981f2118, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110981f2118, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc050ef3e00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b5ec8618, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc062cdf3e0}, 0xc050ef3e00, 0xc066423ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc062cdf3e0}, 0xc050ef3e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110981f2118, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110981f2118, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cdf350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdf350}, {0xc0329993f8, 0x660089}, 0xc032999400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cdf350}, {0xc083930af0}, 0xc050ef3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cdf2c0}, 0xc050ef3c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}, 0xc06ac12d60, 0xc049096420) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}, 0x3, 0xc0b08ec440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc032999980}}}, {0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}, 0xc06ac12d60, 0xc0b08ec440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cdf2c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdf2c0}, {0x40, 0xc032999b70}, 0xc032999ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cdf2c0}, {0x4e9d7a0, 0xc050ef3c00}, 0xc06ac12d60, 0xc049096420) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cdf2c0}, 0xc085102240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f9200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f9200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811136 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7eba40, {0x63c85b8, 0xc052fbec60}, 0xc0d4ac9038, 0x1, 0xc012af50a8, 0x519bbc, 0xc038866180, 0xc06e14c060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc052fbec60}, 0x120, 0xc012af5230, 0x38866120, 0x0, 0x660069, 0xc038866180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7eba40, {0x63c85b8, 0xc052fbec60}, 0xc038866120, {{{{0x0, 0x0}, {0xc066b825a0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc052fbec60}, 0x6352410, 0x537e70a8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02d86de00, {0x63c85b8, 0xc052fbec60}, {0xc038866000, {0x1743011077be564e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02d86de00, {0x63c85b8, 0xc052fbec60}, 0xc04e917360, 0xc012af59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc052fbec60}, 0x660089, {0xc038866000, {0x1743011077be564e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e839c00, {0x63c85b8, 0xc052fbec60}, 0xc02dcf8000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011077be564e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011077be564e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc052fbebd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052fbebd0}, {0xc012af73f8, 0x660089}, 0xc012af7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc052fbebd0}, {0xc0015221e0}, 0xc0a2435e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc052fbeb40}, 0xc0a2435e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}, 0xc04a25be00, 0xc0a97ece10) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}, 0x3, 0xc0c72f9980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc012af7980}}}, {0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}, 0xc04a25be00, 0xc0c72f9980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc052fbeb40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc052fbeb40}, {0x40, 0xc012af7b70}, 0xc012af7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc052fbeb40}, {0x4e9d7a0, 0xc0a2435e00}, 0xc04a25be00, 0xc0a97ece10) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc052fbeb40}, 0xc03049a780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc077433d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc077433d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809485 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01e78ac80, {0x63c85b8, 0xc01ea8f110}, 0xc066a05cc8, 0x1, 0xc01ad4f0a8, 0x519bbc, 0xc07aa0d020, 0xc086e37020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01ea8f110}, 0x120, 0xc01ad4f230, 0x7aa0cfc0, 0x0, 0x660069, 0xc07aa0d020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01e78ac80, {0x63c85b8, 0xc01ea8f110}, 0xc07aa0cfc0, {{{{0x0, 0x0}, {0xc04c36e7e0, 0x9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01ea8f110}, 0x6352410, 0x570853a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0180e3bd0, {0x63c85b8, 0xc01ea8f110}, {0xc07aa0cea0, {0x1743011064fdff23, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0180e3bd0, {0x63c85b8, 0xc01ea8f110}, 0xc065099ea0, 0xc01ad4f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01ea8f110}, 0x660089, {0xc07aa0cea0, {0x1743011064fdff23, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e7d3000, {0x63c85b8, 0xc01ea8f110}, 0xc05f0d2500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011064fdff23, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011064fdff23, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01ea8f080}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ea8f080}, {0xc01ad513f8, 0x660089}, 0xc01ad51400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01ea8f080}, {0xc0015221e0}, 0xc05f0d2300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01ea8eff0}, 0xc05f0d2300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}, 0xc02413d0c0, 0xc041f28a98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}, 0x3, 0xc075816380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01ad51980}}}, {0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}, 0xc02413d0c0, 0xc075816380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01ea8eff0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01ea8eff0}, {0x40, 0xc01ad51b70}, 0xc01ad51ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01ea8eff0}, {0x4e9d7a0, 0xc05f0d2300}, 0xc02413d0c0, 0xc041f28a98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01ea8eff0}, 0xc068537e00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc087900000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc087900000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805764 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc02f7f5860}, 0xc064a3aa00, 0xc0564a0e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02f7f5860}, 0xc064a3aa00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104cf73160, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104cf73160, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f7f57d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f7f57d0}, {0xc01a1687f8, 0x660089}, 0xc01a168800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f7f57d0}, {0xc00102e4e0}, 0xc064a3a900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f7f5770}, 0xc064a3a900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0575991e0, {0x63c85b8, 0xc02f7f55f0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0ab387900, {0x63c85b8, 0xc02f7f55f0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0ab387900, {0x63c8510, 0xc0482add00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc0ab387900, {0x63c8510, 0xc0482add00}, {0x63f7db8, 0x9bf6f48}, 0xc01a16d088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0ab387900, {0x63c85b8, 0xc070c9dc80}, {{0x63f7db8, 0x9bf6f48}, {0xc047075bc4, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0ab387900, {0x63c85b8, 0xc070c9dc80}, {{0x63f7db8, 0x9bf6f48}, {0xc047075bc4, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc047075bc4, 0x6}, 0x0, 0x0}, {0x31f92601, 0xedb7a834f, 0x0}, {0x31f92cc9, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0ab387900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0ab387900, {0x63c8510, 0xc06e014600}, 0xc06f207dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0790a21c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06e014600}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0790a21c0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219687821 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc020435400, {0x63c85b8, 0xc02ba20d20}, 0xc0760d6f00, 0xc06477d900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020435400, {0x63c85b8, 0xc02ba20d20}, 0xc0760d6f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cffdd45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cffdd45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02ba20b10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02ba20b10}, {0xc0908167f8, 0x660089}, 0xc090816800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02ba20b10}, {0xc00102e4e0}, 0xc0760d6e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02ba20930}, 0xc0760d6e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc060435ef0, {0x63c85b8, 0xc02ba20270}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc03a102000, {0x63c85b8, 0xc02ba20270}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc03a102000, {0x63c8510, 0xc04034e0c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc03a102000, {0x63c8510, 0xc04034e0c0}, {0x63f7db8, 0x9bf6f48}, 0xc09081b088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc03a102000, {0x63c85b8, 0xc02f062f60}, {{0x63f7db8, 0x9bf6f48}, {0xc0901d62fd, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc03a102000, {0x63c85b8, 0xc02f062f60}, {{0x63f7db8, 0x9bf6f48}, {0xc0901d62fd, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0901d62fd, 0x6}, 0x0, 0x0}, {0x11f1f164, 0xedb7a834f, 0x0}, {0x11f1f812, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03a102000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03a102000, {0x63c8510, 0xc067895740}, 0xc087b7bdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01fe94bb0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc067895740}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc01fe94bb0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802257 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01e147000, {0x63c85b8, 0xc06ce12e10}, 0xc048ee6a00, 0xc02e903c20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e147000, {0x63c85b8, 0xc06ce12e10}, 0xc048ee6a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110235dc02e, 0x0, 0x0}, 0xc065ad1f80, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110235dc02e, 0x0, 0x0}, 0xc065ad1f80, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ce12d80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ce12d80}, {0xc0768ef918, 0x660089}, 0xc0768ef920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ce12d80}, {0xc00102e4e0}, 0xc048ee6800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06ce12d20}, 0xc048ee6800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc0306b2000}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc06ce12c90}, {0xc00dfd36b0, 0x24}, 0x46c7bd58d, 0xc0387f9d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x4c1af3}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc06ce12c90}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219797010 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc029f2b620}, 0xc0918d8cb0, {0x646cc48, 0xc029f2b680}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc029f2b620}, 0xedb7a834e, 0xc074c00ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc029f2b620}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc013aa9e90}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc0918d8ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc013aa9e90}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802491 [runnable]: sync.runtime_SemacquireMutex(0xc091e3a690, 0x8f, 0x7fd451271200) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc091e3a538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc091e3a560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc05480c900, {0xffffffffffffff15, 0xc091e3a6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc05480c900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc05480c900, {0xc030533d00, 0x32, 0x32}, {0xc030533c00, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092ea2398, {0xc030533d00, 0x32, 0x32}, {0xc030533c00, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092ea0cc0, {0xc030533c00, 0x32, 0x32}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092ea0cc0, {0xc030533d00, 0x32, 0xc030533c00}, {0xc030533c00, 0xc06b2d56c0, 0x31}, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092ea0a00, {0xc030533c00, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0ac0ee3a8, {{0xc06b2d56c0, 0x31, 0x40}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc03381c300, {{0xc02971e1b0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc058be1800, {0x63c85b8, 0xc06ad580c0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06ad580c0}, {0x64a6560, 0xc03381c300}, {0xc02971e1b0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06ad580c0}, {0x6466ed8, 0xc0ac0ee100}, {0xc02971e1b0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0264be810}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01fdf5900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f7a000, {0x63c85b8, 0xc06ad580c0}, 0xc01fdf5900, 0xc068976e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f7a000, {0x63c85b8, 0xc06ad580c0}, 0xc01fdf5900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ad58030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ad58030}, {0xc01fad48f0, 0x660089}, 0xc01fad48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ad58030}, {0x24}, 0xc01fdf5800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c981fb0}, 0xc01fdf5800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc04c7e70e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809195 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0216b8400, {0x63c85b8, 0xc01e0adef0}, 0xc060cca500, 0xc08ca88be0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0216b8400, {0x63c85b8, 0xc01e0adef0}, 0xc060cca500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011064025d5f, 0x0, 0x0}, 0xc067c4a160, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011064025d5f, 0x0, 0x0}, 0xc067c4a160, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e0ade60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e0ade60}, {0xc00e36d3f8, 0x660089}, 0xc00e36d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e0ade60}, {0x18a03d582fe2000}, 0xc060cca300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e0addd0}, 0xc060cca300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}, 0xc01c862f60, 0xc0595797b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}, 0x3, 0xc006589440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e36d980}}}, {0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}, 0xc01c862f60, 0xc006589440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01e0addd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e0addd0}, {0x40, 0xc00e36db70}, 0xc00e36dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01e0addd0}, {0x4e9d7a0, 0xc060cca300}, 0xc01c862f60, 0xc0595797b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01e0addd0}, 0xc0275342a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052b78ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc052b78ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805695 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ed19800, {0x63c85b8, 0xc0459416e0}, 0xc049638300, 0xc072aedb80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ed19800, {0x63c85b8, 0xc0459416e0}, 0xc049638300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011049540436, 0x0, 0x0}, 0xc04eb49650, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011049540436, 0x0, 0x0}, 0xc04eb49650, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0459415f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0459415f0}, {0xc070135918, 0x660089}, 0xc070135920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0459415f0}, {0x24}, 0xc049638000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045941590}, 0xc049638000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc02070b6e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc045941500}, {0xc00dfd36b0, 0x24}, 0x46c7c1794, 0xc088194d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x63c85b8, 0xc03fdbdc20}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc045941500}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805772 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ad17040, {0x63c85b8, 0xc065f68ed0}, 0xc08bb52e10, 0x1, 0x708017e89fdf82, 0x37e11d600, 0xc0384a6420, 0xc071535bc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc073a70fc0, {0x63c85b8, 0xc065f68ed0}, 0xc0384a6420, 0xc00679f1a8, 0x3a, 0x50, 0x0, 0xc0384a6420, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ad17040, {0x63c85b8, 0xc065f68ed0}, 0xc0384a6360, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0154f6d80, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc065f68ed0}, 0x63521a0, 0x4040cb58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a557f90, {0x63c85b8, 0xc065f68ed0}, {0xc0384a6240, {0x17430110026baf99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a557f90, {0x63c85b8, 0xc065f68ed0}, 0xc0740843c0, 0xc00679f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc065f68ed0}, 0x660089, {0xc0384a6240, {0x17430110026baf99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ad30800, {0x63c85b8, 0xc065f68ed0}, 0xc053e13500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110026baf99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110026baf99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x8, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc065f68e40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc065f68e40}, {0xc0067a13f8, 0x660089}, 0xc0067a1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc065f68e40}, {0xc0015221e0}, 0xc053e13200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc065f68d80}, 0xc053e13200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}, 0xc029f23240, 0xc08bb52de0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}, 0x3, 0xc0068c6e00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0067a1980}}}, {0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}, 0xc029f23240, 0xc0068c6e00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc065f68d80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc065f68d80}, {0x40, 0xc0067a1b70}, 0xc0067a1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc065f68d80}, {0x4e9d7a0, 0xc053e13200}, 0xc029f23240, 0xc08bb52de0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc065f68d80}, 0xc05efd0f60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0384a6120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0384a6120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219726890 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af2a248, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc064f0af80, 0xc0befe4800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc064f0af80, {0xc0befe4800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc064f0af80, {0xc0befe4800, 0x735, 0xc0415112c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03a1968e8, {0xc0befe4800, 0x7eda79, 0xc0befe4805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0337a7920, {0xc0befe4800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0647f21b0, {0xc0befe4800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0478b7af8, {0x631b900, 0xc0647f21b0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0478b7880, {0x7fe42ab59310, 0xc0337a7920}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0478b7880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0478b7880, {0xc088bbb000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02e51e660, {0xc088bbb000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc029c0c088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc029c0c088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc029c0c660, {0x634aac0, 0xc029c0c088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc029c0c000, 0xc0826b1100, 0xc09691742b, 0xc09691742d, {0x635bec0, 0xc028214c60}, 0xc000d6ae00, 0xc02a20bda8, 0xc096917429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc029c0c000, {0x63c85b8, 0xc048846de0}, 0xc0826b10f0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc078622020, 0x8}}, 0x0, 0xc048846d80, 0xc048846db0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc048846de0}, {0x643d810, 0xc0337a7920}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc04a66ad50}, {0x643d810, 0xc0337a7920}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc04a66ad50}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810132 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00cddba40, {0x63c85b8, 0xc03a3596e0}, 0xc03d4ae228, 0x1, 0xc00dcd50a8, 0x519bbc, 0xc05b0641e0, 0xc04583c0c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01a3e66c0, {0x63c85b8, 0xc03a3596e0}, 0xc05b0641e0, 0xc00dcd51a8, 0x2c, 0x30, 0x0, 0xc05b0641e0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00cddba40, {0x63c85b8, 0xc03a3596e0}, 0xc05b064120, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0648c27e0, 0x3}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03a3596e0}, 0x6352260, 0x412ac1c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc011c0c960, {0x63c85b8, 0xc03a3596e0}, {0xc05b064000, {0x174301102edfba16, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc011c0c960, {0x63c85b8, 0xc03a3596e0}, 0xc00b5b26e0, 0xc00dcd59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03a3596e0}, 0x660089, {0xc05b064000, {0x174301102edfba16, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0122d9400, {0x63c85b8, 0xc03a3596e0}, 0xc0522f1a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102edfba16, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102edfba16, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3e, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03a359620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a359620}, {0xc00dcd73f8, 0x660089}, 0xc00dcd7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03a359620}, {0x6d72657275746361}, 0xc0522f1800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a359470}, 0xc0522f1800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}, 0xc050ea96c0, 0xc01bb1eed0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}, 0x3, 0xc0996fb4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00dcd7980}}}, {0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}, 0xc050ea96c0, 0xc0996fb4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03a359470}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a359470}, {0x40, 0xc00dcd7b70}, 0xc00dcd7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03a359470}, {0x4e9d7a0, 0xc0522f1800}, 0xc050ea96c0, 0xc01bb1eed0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03a359470}, 0xc024f06960, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07649de60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07649de60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810050 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01876d900, {0x63c85b8, 0xc0235253e0}, 0xc02eae2a80, 0x1, 0x7080180604051f, 0x37e11d600, 0xc054f243c0, 0xc071b29020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0235253e0}, 0x120, 0xc01a371230, 0x54f24360, 0x0, 0x660069, 0xc054f243c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01876d900, {0x63c85b8, 0xc0235253e0}, 0xc054f24360, {{{{0x0, 0x0}, {0xc0543e77a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0235253e0}, 0x6352410, 0x3102f7e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016e75f40, {0x63c85b8, 0xc0235253e0}, {0xc054f24120, {0x1743011068a4d230, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016e75f40, {0x63c85b8, 0xc0235253e0}, 0xc072e68000, 0xc01a3719f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0235253e0}, 0x660089, {0xc054f24120, {0x1743011068a4d230, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc018944c00, {0x63c85b8, 0xc0235253e0}, 0xc01c6fbd00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011068a4d230, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011068a4d230, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023525350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023525350}, {0xc01a3733f8, 0x660089}, 0xc01a373400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023525350}, {0x0}, 0xc01c6fb600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0235252c0}, 0xc01c6fb600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}, 0xc05f82d680, 0xc02eae2a50) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}, 0x3, 0xc064e5ed40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01a373980}}}, {0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}, 0xc05f82d680, 0xc064e5ed40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0235252c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0235252c0}, {0x40, 0xc01a373b70}, 0xc01a373ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0235252c0}, {0x4e9d7a0, 0xc01c6fb600}, 0xc05f82d680, 0xc02eae2a50) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0235252c0}, 0xc0459f1ec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0a3ff7e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0a3ff7e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812189 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc073776870}, 0xc0204cd1e8, 0x1, 0x7080181d7aa0ca, 0x37e11d600, 0xc013f31a40, 0xc091397020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc073776870}, 0x120, 0xc039d49230, 0x13f319e0, 0x0, 0x660069, 0xc013f31a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc073776870}, 0xc013f319e0, {{{{0x0, 0x0}, {0xc02875a900, 0x35}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073776870}, 0x6352410, 0x178389a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc073776870}, {0xc013f318c0, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc073776870}, 0xc0a91f3040, 0xc039d499f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073776870}, 0x660089, {0xc013f318c0, {0x174301107a04e32d, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc073776870}, 0xc06d231100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107a04e32d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0737767e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0737767e0}, {0xc039d4b3f8, 0x660089}, 0xc039d4b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0737767e0}, {0x7fe42aa84a70}, 0xc06d230e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073776750}, 0xc06d230e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}, 0xc0486a7920, 0xc0204cd1b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}, 0x3, 0xc05750b480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc039d4b980}}}, {0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}, 0xc0486a7920, 0xc05750b480) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc073776750}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073776750}, {0x40, 0xc039d4bb70}, 0xc039d4bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc073776750}, {0x4e9d7a0, 0xc06d230e00}, 0xc0486a7920, 0xc0204cd1b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc073776750}, 0xc018b18300, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f31440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc013f31440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809177 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc000e4e800, {0x63c85b8, 0xc046316ae0}, 0xc05bac0f00, 0xc0689765a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000e4e800, {0x63c85b8, 0xc046316ae0}, 0xc05bac0f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063e21f99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063e21f99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046316a50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046316a50}, {0xc0209953f8, 0x660089}, 0xc020995400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046316a50}, {0x7fe4292605d0}, 0xc05bac0d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046316990}, 0xc05bac0d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}, 0xc03f4997a0, 0xc042c25ab8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}, 0x3, 0xc04f1cbd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc020995980}}}, {0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}, 0xc03f4997a0, 0xc04f1cbd00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046316990}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046316990}, {0x40, 0xc020995b70}, 0xc020995ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046316990}, {0x4e9d7a0, 0xc05bac0d00}, 0xc03f4997a0, 0xc042c25ab8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046316990}, 0xc03de0bf20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06e4b1440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06e4b1440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811445 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc019d41830}, 0xc0251866a8, 0x1, 0x7080181502b96a, 0x37e11d600, 0xc06beafa40, 0xc04eae9920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc019d41830}, 0x120, 0xc0168dd230, 0x6beaf9e0, 0x0, 0x660069, 0xc06beafa40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc019d41830}, 0xc06beaf9e0, {{{{0x0, 0x0}, {0xc079260240, 0x24}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc019d41830}, 0x6352410, 0x33df4568) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc019d41830}, {0xc06beaf8c0, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc019d41830}, 0xc06cd2ff40, 0xc0168dd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc019d41830}, 0x660089, {0xc06beaf8c0, {0x174301105add9208, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc019d41830}, 0xc05758ca00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105add9208, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc019d41770}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019d41770}, {0xc0168df3f8, 0x660089}, 0xc0168df400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc019d41770}, {0x7fe424ae0a00}, 0xc05758c800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc019d416b0}, 0xc05758c800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}, 0xc0191677a0, 0xc0251865d0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}, 0x3, 0xc00dd3c380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0168df980}}}, {0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}, 0xc0191677a0, 0xc00dd3c380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc019d416b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc019d416b0}, {0x40, 0xc0168dfb70}, 0xc0168dfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc019d416b0}, {0x4e9d7a0, 0xc05758c800}, 0xc0191677a0, 0xc0251865d0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc019d416b0}, 0xc03148f260, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04ffce120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04ffce120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791847 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0570d56b0, {0x63c85b8, 0xc0399002a0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc0ea904c00, {0x63c85b8, 0xc0399002a0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc0ea904c00, {0x63c8510, 0xc04c2e1580}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc0ea904c00, {0x63c8510, 0xc04c2e1580}, 0xc062493cd0, {0x7fe42b790cf0, 0xc02afec6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0ea904c00, {0x63c85b8, 0xc026e3d2f0}, {{0x63f88f8, 0xc062493cd0}, {0xc097ed7acf, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0ea904c00, {0x63c85b8, 0xc026e3d2f0}, {{0x63f88f8, 0xc062493cd0}, {0xc097ed7acf, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc062493cd0}, {0xc097ed7acf, 0x23}, 0x0, 0x0}, {0x2a15f4c, 0xedb7a834f, 0x0}, {0x2a16747, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0ea904c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0ea904c00, {0x63c8510, 0xc067cc0cc0}, 0xc0768eddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc006b654a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc067cc0cc0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc006b654a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219803765 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc030124000, {0x63c85b8, 0xc0457853e0}, 0xc064951a00, 0xc04e572140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030124000, {0x63c85b8, 0xc0457853e0}, 0xc064951a00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fb966a5, 0x0, 0x0}, 0xc0520825e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fb966a5, 0x0, 0x0}, 0xc0520825e0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045785350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045785350}, {0xc04ed173f8, 0x660089}, 0xc04ed17400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045785350}, {0x7fe13d162a38}, 0xc064951700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0457852c0}, 0xc064951700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}, 0xc01bc5e5a0, 0xc06d613638) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}, 0x3, 0xc0d3fd8a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04ed17980}}}, {0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}, 0xc01bc5e5a0, 0xc0d3fd8a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0457852c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0457852c0}, {0x40, 0xc04ed17b70}, 0xc04ed17ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0457852c0}, {0x4e9d7a0, 0xc064951700}, 0xc01bc5e5a0, 0xc06d613638) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0457852c0}, 0xc0b5314240, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc047a70b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc047a70b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812522 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807332 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0357c0, {0x63c85b8, 0xc02f6d18c0}, 0xc044b1a048, 0x1, 0xc002edb0a8, 0x519bbc, 0xc092ce9740, 0xc066f8d1a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc073da6c60, {0x63c85b8, 0xc02f6d18c0}, 0xc073da6c90, 0xc002edb1a8, 0x41, 0x60, 0xffffffffffffffff, 0xc092ce9740, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0357c0, {0x63c85b8, 0xc02f6d18c0}, 0xc092ce9500, {{{{0xc06118dd40, 0x1}, {0xc05c772120, 0x1}}, {{0xc051d39440, 0x5}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02f6d18c0}, 0x63521a0, 0x25ab6cd0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfdd770, {0x63c85b8, 0xc02f6d18c0}, {0xc07c0dea20, {0x1743010ffbec6594, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfdd770, {0x63c85b8, 0xc02f6d18c0}, 0xc0772d8dc0, 0xc002edb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02f6d18c0}, 0x660089, {0xc07c0dea20, {0x1743010ffbec6594, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06c800, {0x63c85b8, 0xc02f6d18c0}, 0xc08e4f2c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffbec6594, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffbec6594, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02f6d1590}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f6d1590}, {0xc002edd3f8, 0x660089}, 0xc002edd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02f6d1590}, {0xc0015221e0}, 0xc08e4f2a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02f6d1440}, 0xc08e4f2a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}, 0xc012abbae0, 0xc082101758) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}, 0x3, 0xc075ce2c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc002edd980}}}, {0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}, 0xc012abbae0, 0xc075ce2c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02f6d1440}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02f6d1440}, {0x40, 0xc002eddb70}, 0xc002eddab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02f6d1440}, {0x4e9d7a0, 0xc08e4f2a00}, 0xc012abbae0, 0xc082101758) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02f6d1440}, 0xc029b0b9e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07c0de480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07c0de480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803829 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0214da000, {0x63c85b8, 0xc07e1baa20}, 0xc097368400, 0xc08f98aa00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc07e1baa20}, 0xc097368400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032171205, 0x0, 0x0}, 0xc0a0050cf0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032171205, 0x0, 0x0}, 0xc0a0050cf0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07e1ba990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e1ba990}, {0xc071ab13f8, 0x660089}, 0xc071ab1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07e1ba990}, {0x7fe426a11c30}, 0xc097368200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07e1ba900}, 0xc097368200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}, 0xc0606eb6a0, 0xc038313e30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}, 0x3, 0xc02661c780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc071ab1980}}}, {0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}, 0xc0606eb6a0, 0xc02661c780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07e1ba900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e1ba900}, {0x40, 0xc071ab1b70}, 0xc071ab1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07e1ba900}, {0x4e9d7a0, 0xc097368200}, 0xc0606eb6a0, 0xc038313e30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07e1ba900}, 0xc036292b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0520ae120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809175 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc07bc0fb60}, 0xc03c1d4bd0, 0x1, 0xc017ecd0a8, 0x519bbc, 0xc06e4b0960, 0xc08d5ff920) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07bc0fb60}, 0x120, 0xc017ecd230, 0x6e4b0900, 0x0, 0x660069, 0xc06e4b0960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc07bc0fb60}, 0xc06e4b0900, {{{{0x0, 0x0}, {0xc09f556000, 0x2b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07bc0fb60}, 0x6352410, 0x1b308328) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc07bc0fb60}, {0xc06e4b07e0, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc07bc0fb60}, 0xc03b364640, 0xc017ecd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07bc0fb60}, 0x660089, {0xc06e4b07e0, {0x174301104b3991ad, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc07bc0fb60}, 0xc05bac0500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104b3991ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07bc0fad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bc0fad0}, {0xc017ecf3f8, 0x660089}, 0xc017ecf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07bc0fad0}, {0x0}, 0xc05bac0200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07bc0fa40}, 0xc05bac0200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}, 0xc03f499540, 0xc042c25470) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}, 0x3, 0xc04f1cb640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc017ecf980}}}, {0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}, 0xc03f499540, 0xc04f1cb640) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07bc0fa40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bc0fa40}, {0x40, 0xc017ecfb70}, 0xc017ecfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07bc0fa40}, {0x4e9d7a0, 0xc05bac0200}, 0xc03f499540, 0xc042c25470) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07bc0fa40}, 0xc03de0b560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06e4b06c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06e4b06c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808597 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020d48280, {0x63c85b8, 0xc02d01b290}, 0xc029af26f0, 0x1, 0xc00caf30a8, 0x519bbc, 0xc06425b260, 0xc051868ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc02d01b290}, 0x120, 0xc00caf3230, 0x6425b200, 0x0, 0x660069, 0xc06425b260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020d48280, {0x63c85b8, 0xc02d01b290}, 0xc06425b200, {{{{0x0, 0x0}, {0xc033690120, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02d01b290}, 0x6352410, 0x324316f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00a63def0, {0x63c85b8, 0xc02d01b290}, {0xc06425b0e0, {0x174301105b709713, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00a63def0, {0x63c85b8, 0xc02d01b290}, 0xc04ed195e0, 0xc00caf39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02d01b290}, 0x660089, {0xc06425b0e0, {0x174301105b709713, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001920400, {0x63c85b8, 0xc02d01b290}, 0xc04f7e8100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b709713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b709713, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02d01b140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d01b140}, {0xc00caf53f8, 0x660089}, 0xc00caf5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02d01b140}, {0x7fe4257489a0}, 0xc0c8bdbf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02d01b020}, 0xc0c8bdbf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}, 0xc05159a3e0, 0xc03592c198) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}, 0x3, 0xc058e09440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00caf5980}}}, {0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}, 0xc05159a3e0, 0xc058e09440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02d01b020}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02d01b020}, {0x40, 0xc00caf5b70}, 0xc00caf5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02d01b020}, {0x4e9d7a0, 0xc0c8bdbf00}, 0xc05159a3e0, 0xc03592c198) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02d01b020}, 0xc02ac2ef00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05b6d9680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc05b6d9680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807623 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc041376c30}, 0xc06292e918, 0x1, 0xc0066e70a8, 0x519bbc, 0xc0579f4cc0, 0xc07952aba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc041376c30}, 0x120, 0xc0066e7230, 0x579f4c60, 0x0, 0x660069, 0xc0579f4cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc041376c30}, 0xc0579f4c60, {{{{0x0, 0x0}, {0xc07bc0c000, 0x23}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc041376c30}, 0x6352410, 0x53415950) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc041376c30}, {0xc0579f4b40, {0x1743011058606be6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc041376c30}, 0xc049b11400, 0xc0066e79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc041376c30}, 0x660089, {0xc0579f4b40, {0x1743011058606be6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc041376c30}, 0xc0997f9400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011058606be6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011058606be6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041376ba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041376ba0}, {0xc0066e93f8, 0x660089}, 0xc0066e9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041376ba0}, {0x0}, 0xc0997f9200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041376b10}, 0xc0997f9200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}, 0xc086665620, 0xc02385d830) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}, 0x3, 0xc061b67800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0066e9980}}}, {0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}, 0xc086665620, 0xc061b67800) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041376b10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041376b10}, {0x40, 0xc0066e9b70}, 0xc0066e9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041376b10}, {0x4e9d7a0, 0xc0997f9200}, 0xc086665620, 0xc02385d830) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041376b10}, 0xc02f8cd8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0579f4a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0579f4a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219798828 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc093112548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0931125b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc092a0fbf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc081ffb880, {0xc08ca03b80, 0x2b, 0x48}, {0xc062a6ac40, 0x2b, 0x6f}, 0x1, 0xfb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092a0fa88, {0xc08ca03b80, 0x1, 0x6f}, {0xc062a6ac40, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092a0fa88, {0xc08ca03b80, 0x2b, 0x48}, {0xc062a6ac40, 0x2b, 0x6f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092a0f1c0, {0xc062a6ac40, 0x6f, 0x48}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092a0f1c0, {0xc08ca03b80, 0x2b, 0xc062a6ac40}, {0xc062a6ac40, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092a0ef00, {0xc062a6ac40, 0x2b, 0x6f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc06060e588, {{0xc028dc3380, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc081c6f700, {{0xc028dc3380, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc030815c00, {0x63c85b8, 0xc01f43cc60}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc01f43cc60}, {0x64a6560, 0xc081c6f700}, {0xc028dc3380, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc01f43cc60}, {0x6466ed8, 0xc06060e580}, {0xc028dc3380, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc074391428}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc092aa3000, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02057d400, {0x63c85b8, 0xc01f43cc60}, 0xc092aa3000, 0xc0895b52c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02057d400, {0x63c85b8, 0xc01f43cc60}, 0xc092aa3000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01f43cb70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01f43cb70}, {0xc09365a8f0, 0x660089}, 0xc09365a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01f43cb70}, {0xc00102e4e0}, 0xc092aa2f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01f43cb10}, 0xc092aa2f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0738fc300}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219727453 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc00e78a9d0, 0x116) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc03bf64a80) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc03bf650e0) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04f01f900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04f01f900, {0x63c8510, 0xc00e78aa00}, 0xc06e69fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07c0f9150) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc00e78aa00}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07c0f9150) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219804550 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e099000, {0x63c85b8, 0xc03a014720}, 0xc0a4f85400, 0xc06f10c3c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e099000, {0x63c85b8, 0xc03a014720}, 0xc0a4f85400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110306734ae, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03a014690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a014690}, {0xc0750453f8, 0x660089}, 0xc075045400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03a014690}, {0xc010530970}, 0xc0a4f85200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03a014540}, 0xc0a4f85200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}, 0xc011ef9d80, 0xc028551398) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}, 0x3, 0xc044f493c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc075045980}}}, {0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}, 0xc011ef9d80, 0xc044f493c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03a014540}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03a014540}, {0x40, 0xc075045b70}, 0xc075045ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03a014540}, {0x4e9d7a0, 0xc0a4f85200}, 0xc011ef9d80, 0xc028551398) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03a014540}, 0xc06be24fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc060b3eea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc060b3eea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801375 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03c908c80, {0x63c85b8, 0xc076695170}, 0xc02e5e0558, 0x1, 0xc0521910a8, 0x519bbc, 0xc07d48dec0, 0xc045d88f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc076695170}, 0x120, 0xc052191230, 0x7d48de60, 0x0, 0x660069, 0xc07d48dec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03c908c80, {0x63c85b8, 0xc076695170}, 0xc07d48de60, {{{{0x0, 0x0}, {0xc079be8240, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc076695170}, 0x6352410, 0x1a06e610) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc00852fe50, {0x63c85b8, 0xc076695170}, {0xc07d48dd40, {0x174301101ae98db8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc00852fe50, {0x63c85b8, 0xc076695170}, 0xc048082dc0, 0xc0521919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc076695170}, 0x660089, {0xc07d48dd40, {0x174301101ae98db8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b078c00, {0x63c85b8, 0xc076695170}, 0xc0a3e71600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ae98db8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ae98db8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0766950e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0766950e0}, {0xc0521933f8, 0x660089}, 0xc052193400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0766950e0}, {0xc010928558}, 0xc0a3e71400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076695050}, 0xc0a3e71400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}, 0xc00bdabce0, 0xc042de39c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}, 0x3, 0xc0cb6ea740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc052193980}}}, {0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}, 0xc00bdabce0, 0xc0cb6ea740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076695050}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076695050}, {0x40, 0xc052193b70}, 0xc052193ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076695050}, {0x4e9d7a0, 0xc0a3e71400}, 0xc00bdabce0, 0xc042de39c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076695050}, 0xc0384810e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07d48dc20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc07d48dc20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807035 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc025428d80}, 0xc039aa3cc8, 0x1, 0xc00541f0a8, 0x519bbc, 0xc09b1664e0, 0xc09bbf5380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc025428d80}, 0x120, 0xc00541f230, 0x9b166480, 0x0, 0x660069, 0xc09b1664e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc025428d80}, 0xc09b166480, {{{{0x0, 0x0}, {0xc028ef4240, 0x116}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc025428d80}, 0x6352410, 0x25ab69b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc025428d80}, {0xc09b166360, {0x17430110558aa4be, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc025428d80}, 0xc02f05c6e0, 0xc00541f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc025428d80}, 0x660089, {0xc09b166360, {0x17430110558aa4be, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc025428d80}, 0xc08bf6a900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110558aa4be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110558aa4be, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc025428cf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc025428cf0}, {0xc0054213f8, 0x660089}, 0xc005421400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc025428cf0}, {0x7fe3489f7cb8}, 0xc08bf6a700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc025428a20}, 0xc08bf6a700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}, 0xc05665f6c0, 0xc039aa3f98) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}, 0x3, 0xc0379ed040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005421980}}}, {0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}, 0xc05665f6c0, 0xc0379ed040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc025428a20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc025428a20}, {0x40, 0xc005421b70}, 0xc005421ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc025428a20}, {0x4e9d7a0, 0xc08bf6a700}, 0xc05665f6c0, 0xc039aa3f98) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc025428a20}, 0xc0911dad20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0592d0fc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0592d0fc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801964 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0311ef800, {0x63c85b8, 0xc01fe64510}, 0xc0a570f500, 0xc039b4bea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0311ef800, {0x63c85b8, 0xc01fe64510}, 0xc0a570f500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff497f710, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff497f710, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fe64480}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fe64480}, {0xc00eb273f8, 0x660089}, 0xc00eb27400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fe64480}, {0x7fe3b672c188}, 0xc0a570f300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fe64240}, 0xc0a570f300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}, 0xc0621ff600, 0xc09d385b30) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}, 0x3, 0xc00f24f100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00eb27980}}}, {0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}, 0xc0621ff600, 0xc00f24f100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01fe64240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fe64240}, {0x40, 0xc00eb27b70}, 0xc00eb27ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01fe64240}, {0x4e9d7a0, 0xc0a570f300}, 0xc0621ff600, 0xc09d385b30) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01fe64240}, 0xc029053c80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04cb0e360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc04cb0e360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812759 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805923 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021c5a8c0, {0x63c85b8, 0xc054204c90}, 0xc026b78e28, 0x1, 0xc0055f50a8, 0x519bbc, 0xc055cc8240, 0xc06e0e71a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc09cc32900, {0x63c85b8, 0xc054204c90}, 0xc055cc8240, 0xc0055f51a8, 0x30, 0x40, 0x1, 0xc055cc8240, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021c5a8c0, {0x63c85b8, 0xc054204c90}, 0xc055cc8000, {{{{0xc0462517a0, 0x1}, {0xc03c6387e0, 0x1}}, {{0xc057368240, 0x4}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc054204c90}, 0x63521a0, 0x321a0170) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c42780, {0x63c85b8, 0xc054204c90}, {0xc036dd4240, {0x1743010fff7f045b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c42780, {0x63c85b8, 0xc054204c90}, 0xc05a2100a0, 0xc0055f59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc054204c90}, 0x660089, {0xc036dd4240, {0x1743010fff7f045b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c6c800, {0x63c85b8, 0xc054204c90}, 0xc03999fc00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010fff7f045b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010fff7f045b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc054204bd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054204bd0}, {0xc0055f73f8, 0x660089}, 0xc0055f7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc054204bd0}, {0x7fe422bf5ea8}, 0xc03999fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc054204b10}, 0xc03999fa00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}, 0xc09a01ea80, 0xc07416b1a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}, 0x3, 0xc037b49440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0055f7980}}}, {0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}, 0xc09a01ea80, 0xc037b49440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc054204b10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc054204b10}, {0x40, 0xc0055f7b70}, 0xc0055f7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc054204b10}, {0x4e9d7a0, 0xc03999fa00}, 0xc09a01ea80, 0xc07416b1a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc054204b10}, 0xc0c21b64e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc085c8eb40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc085c8eb40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219793141 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc089a6eb00, {0x63c85b8, 0xc03be45a70}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc046976c00, {0x63c85b8, 0xc03be45a70}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc046976c00, {0x63c8510, 0xc03ed875c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc046976c00, {0x63c8510, 0xc03ed875c0}, {0x63f7db8, 0x9bf6f48}, 0xc098455088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc046976c00, {0x63c85b8, 0xc0773c68a0}, {{0x63f7db8, 0x9bf6f48}, {0xc0b9dcc31f, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc046976c00, {0x63c85b8, 0xc0773c68a0}, {{0x63f7db8, 0x9bf6f48}, {0xc0b9dcc31f, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0b9dcc31f, 0x6}, 0x0, 0x0}, {0x36785a5, 0xedb7a834f, 0x0}, {0x3678bad, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc046976c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc046976c00, {0x63c8510, 0xc0286f2000}, 0xc075f49dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03fc920b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0286f2000}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03fc920b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219805586 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016f5da40, {0x63c85b8, 0xc0824da120}, 0xc047826300, 0x1, 0xc094f910a8, 0x519bbc, 0xc05ee07a40, 0xc05c0236e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0824da120}, 0x120, 0xc094f91230, 0x5ee079e0, 0xc05bae2820, 0x18, 0xc05ee07a40, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016f5da40, {0x63c85b8, 0xc0824da120}, 0xc05ee079e0, {{{{0x0, 0x0}, {0xc05529a6c0, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0824da120}, 0x6352410, 0x3d0adf88) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016fda500, {0x63c85b8, 0xc0824da120}, {0xc05ee078c0, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016fda500, {0x63c85b8, 0xc0824da120}, 0xc05a210820, 0xc094f919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0824da120}, 0x660089, {0xc05ee078c0, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017050000, {0x63c85b8, 0xc0824da120}, 0xc0c6d58300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc020bb3f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020bb3f80}, {0xc094f933f8, 0x660089}, 0xc094f93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc020bb3f80}, {0x7fe3b6781b98}, 0xc04a0f3f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc020bb3ef0}, 0xc04a0f3f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}, 0xc01f2e8b00, 0xc04a2ffec0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}, 0x3, 0xc0544d0140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc094f93980}}}, {0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}, 0xc01f2e8b00, 0xc0544d0140) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc020bb3ef0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc020bb3ef0}, {0x40, 0xc094f93b70}, 0xc094f93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}, 0x203015, 0x203015) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc020bb3ef0}, {0x4e9d7a0, 0xc04a0f3f00}, 0xc01f2e8b00, 0xc04a2ffec0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc020bb3ef0}, 0xc04c2502a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05edb7e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05edb7e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802265 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc07beac548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc07beac5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc092e916f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc07156a000, {0xc04f09a6c0, 0x2b, 0x2b}, {0xc04f09a690, 0x2b, 0x2b}, 0x1, 0x16) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc092e91588, {0xc04f09a6c0, 0x1, 0x2b}, {0xc04f09a690, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092e91588, {0xc04f09a6c0, 0x2b, 0x2b}, {0xc04f09a690, 0x2b, 0x2b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092e90cc0, {0xc04f09a690, 0x2b, 0x2b}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092e90cc0, {0xc04f09a6c0, 0x2b, 0xc04f09a690}, {0xc04f09a690, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092e90a00, {0xc04f09a690, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc059a47088, {{0xc047c34480, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04315d200, {{0xc047c34480, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc01d931c00, {0x63c85b8, 0xc08a924db0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc08a924db0}, {0x64a6560, 0xc04315d200}, {0xc047c34480, 0xc9c84a5e6380c2b, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc08a924db0}, {0x6466ed8, 0xc059a47080}, {0xc047c34480, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0560fb890}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04d6f7800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020edfc00, {0x63c85b8, 0xc08a924db0}, 0xc04d6f7800, 0xc0783c7900) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020edfc00, {0x63c85b8, 0xc08a924db0}, 0xc04d6f7800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08a924d20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08a924d20}, {0xc001c988f0, 0x660089}, 0xc001c988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08a924d20}, {0xc00102e4e0}, 0xc04d6f7700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08a924c90}, 0xc04d6f7700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0bb639650}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811860 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024ab5e00, {0x63c85b8, 0xc06f283da0}, 0xc0450fc3a8, 0x1, 0xc018eb30a8, 0x519bbc, 0xc06275f260, 0xc046637da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06f283da0}, 0x120, 0xc018eb3230, 0x6275f200, 0x0, 0x660069, 0xc06275f260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024ab5e00, {0x63c85b8, 0xc06f283da0}, 0xc06275f200, {{{{0x0, 0x0}, {0xc01e9419e0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06f283da0}, 0x6352410, 0x3abf6078) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024c04690, {0x63c85b8, 0xc06f283da0}, {0xc06275f0e0, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024c04690, {0x63c85b8, 0xc06f283da0}, 0xc048c07720, 0xc018eb39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f283da0}, 0x660089, {0xc06275f0e0, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024c29c00, {0x63c85b8, 0xc06f283da0}, 0xc082c3e200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f283d10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f283d10}, {0xc018eb53f8, 0x660089}, 0xc018eb5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f283d10}, {0x7fe423eea220}, 0xc082c3e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f283c80}, 0xc082c3e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}, 0xc039294060, 0xc0450fcca8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}, 0x3, 0xc06566d180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc018eb5980}}}, {0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}, 0xc039294060, 0xc06566d180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f283c80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f283c80}, {0x40, 0xc018eb5b70}, 0xc018eb5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f283c80}, {0x4e9d7a0, 0xc082c3e000}, 0xc039294060, 0xc0450fcca8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f283c80}, 0xc08f62d800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059510c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059510c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219793020 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0654c5020}, 0xc00d528cb0, {0x646cc48, 0xc0654c5080}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0654c5020}, 0xedb7a834e, 0xc04e69d0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0654c5020}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0206b3440}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc00d528ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc0206b3440}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809139 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc040c5f770}, 0xc0565172c0, 0x1, 0xc01ae750a8, 0x519bbc, 0xc06dff8cc0, 0xc0747104e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc040c5f770}, 0x120, 0xc01ae75230, 0x6dff8c60, 0x0, 0x660069, 0xc06dff8cc0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc040c5f770}, 0xc06dff8c60, {{{{0x0, 0x0}, {0xc031d41440, 0x2a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc040c5f770}, 0x6352410, 0x29b7a8c8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc040c5f770}, {0xc06dff8b40, {0x17430110618a8100, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc040c5f770}, 0xc0637da780, 0xc01ae759f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc040c5f770}, 0x660089, {0xc06dff8b40, {0x17430110618a8100, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc040c5f770}, 0xc056dd1800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110618a8100, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110618a8100, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc040c5f6e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc040c5f6e0}, {0xc01ae773f8, 0x660089}, 0xc01ae77400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc040c5f6e0}, {0x494745520a1605e8}, 0xc056dd1600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc040c5f650}, 0xc056dd1600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}, 0xc03c1d00c0, 0xc056517548) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}, 0x3, 0xc072991500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01ae77980}}}, {0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}, 0xc03c1d00c0, 0xc072991500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc040c5f650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc040c5f650}, {0x40, 0xc01ae77b70}, 0xc01ae77ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc040c5f650}, {0x4e9d7a0, 0xc056dd1600}, 0xc03c1d00c0, 0xc056517548) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc040c5f650}, 0xc053046660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06dff8a20, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06dff8a20, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812016 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc049633b00}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802891 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02c84c400, {0x63c85b8, 0xc01bb00060}, 0xc029287600, 0xc08767ebe0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c84c400, {0x63c85b8, 0xc01bb00060}, 0xc029287600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102757f154, 0x0, 0x0}, 0xc040262020, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102757f154, 0x0, 0x0}, 0xc040262020, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03bf0ff80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bf0ff80}, {0xc08da97918, 0x660089}, 0xc08da97920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03bf0ff80}, {0xc00102e4e0}, 0xc029287500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03bf0fe90}, 0xc029287500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc066ae4660}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc03bf0fe00}, {0xc00dfd36b0, 0x24}, 0x46c7c18c8, 0xc06366bd40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc0899590e0, 0xc0199d5c78}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc03bf0fe00}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810801 [runnable]: sync.runtime_SemacquireMutex(0x45dbad, 0x0, 0xc075cf2000) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0xc0a20a7c00, 0xc0108b9158) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x4be10c0, 0x4e39320}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0a20a7c00, {0xc02fce0800, 0x32, 0x74}, {0xc082ba6840, 0x32, 0x5e}, 0x0, 0x45) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc070fd4488, {0xc02fce0800, 0x1, 0x5e}, {0xc082ba6840, 0x6d1, 0xc0108b9298}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc070fd4488, {0xc02fce0800, 0x32, 0x74}, {0xc082ba6840, 0x32, 0x5e}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc070fd3bc0, {0xc082ba6840, 0x2, 0x8}, 0x2f, 0xc8) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc070fd3bc0, {0xc02fce0800, 0x5e, 0xc03870e820}, {0xc082ba6840, 0xc03870e820, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc070fd3900, {0xc082ba6840, 0x32, 0x5e}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc086b842a8, {{0xc03870e820, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc083506500, {{0xc071bb0c90, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc037df7000, {0x63c85b8, 0xc0461a51d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0461a51d0}, {0x64a6560, 0xc083506500}, {0xc071bb0c90, 0x0, 0x30}, {0x174301107396cc18, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0461a51d0}, {0x6466ed8, 0xc086b84000}, {0xc071bb0c90, 0x2a, 0x30}, {0x174301107396cc18, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc023f9a300}, {{0x174301107396cc18, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174301107396cc18, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06b6e2700, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743011091643118, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01dd95800, {0x63c85b8, 0xc0461a51d0}, 0xc06b6e2700, 0xc046ded180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd95800, {0x63c85b8, 0xc0461a51d0}, 0xc06b6e2700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107396cc18, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107396cc18, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0461a5140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0461a5140}, {0xc0108bf3f8, 0x660089}, 0xc0108bf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0461a5140}, {0xc0015221e0}, 0xc06b6e2500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0461a50b0}, 0xc06b6e2500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}, 0xc00833df60, 0xc07b35c648) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}, 0x3, 0xc00f6fe9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0108bf980}}}, {0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}, 0xc00833df60, 0xc00f6fe9c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0461a50b0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0461a50b0}, {0x40, 0xc0108bfb70}, 0xc0108bfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0461a50b0}, {0x4e9d7a0, 0xc06b6e2500}, 0xc00833df60, 0xc07b35c648) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0461a50b0}, 0xc0242c7440, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc012e14b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc012e14b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802688 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02ff75c00, {0x63c85b8, 0xc045df6c00}, 0xc0b9ef0000, 0xc087686280) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75c00, {0x63c85b8, 0xc045df6c00}, 0xc0b9ef0000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110247bc7de, 0x0, 0x0}, 0xc03c6a6bf0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110247bc7de, 0x0, 0x0}, 0xc03c6a6bf0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc045df6b70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045df6b70}, {0xc04250f3f8, 0x660089}, 0xc04250f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc045df6b70}, {0x0}, 0xc019ed7d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc045df6ae0}, 0xc019ed7d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}, 0xc03d2535e0, 0xc068a75ec0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}, 0x3, 0xc0204974c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04250f980}}}, {0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}, 0xc03d2535e0, 0xc0204974c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc045df6ae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc045df6ae0}, {0x40, 0xc04250fb70}, 0xc04250fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc045df6ae0}, {0x4e9d7a0, 0xc019ed7d00}, 0xc03d2535e0, 0xc068a75ec0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc045df6ae0}, 0xc021d7fc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cfc0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc04652cfc0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801550 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc021cd8400, {0x63c85b8, 0xc04173b320}, 0xc050943300, 0xc046dedc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021cd8400, {0x63c85b8, 0xc04173b320}, 0xc050943300, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101ceeb95a, 0x0, 0x0}, 0xc00d975e60, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101ceeb95a, 0x0, 0x0}, 0xc00d975e60, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04173b290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04173b290}, {0xc0766cf918, 0x660089}, 0xc0766cf920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04173b290}, {0xc00102e4e0}, 0xc050943200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04173b230}, 0xc050943200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc046cf6cc0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc04173b1a0}, {0xc00dfd36b0, 0x24}, 0x470de556e, 0xc017292d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc09d032270, 0xc00e7a0c78}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc04173b1a0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807727 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fea7000, {0x63c85b8, 0xc07a4152f0}, 0xc04a321e00, 0xc0480832c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc07a4152f0}, 0xc04a321e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011059dc2ef8, 0x0, 0x0}, 0xc08e62b280, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011059dc2ef8, 0x0, 0x0}, 0xc08e62b280, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a415260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a415260}, {0xc00e7333f8, 0x660089}, 0xc00e733400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a415260}, {0x0}, 0xc04a321c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a4151d0}, 0xc04a321c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}, 0xc089407f40, 0xc030d2d350) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}, 0x3, 0xc064bac4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e733980}}}, {0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}, 0xc089407f40, 0xc064bac4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07a4151d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a4151d0}, {0x40, 0xc00e733b70}, 0xc00e733ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07a4151d0}, {0x4e9d7a0, 0xc04a321c00}, 0xc089407f40, 0xc030d2d350) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07a4151d0}, 0xc0651dd980, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc071835560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc071835560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811017 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc0308c9410}, 0xc02ab7f068, 0x1, 0x0, 0x1, 0xc054b9e060, 0xc0748ba840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0308c9410}, 0x120, 0xc015d18728, 0x54b9e000, 0x0, 0x660069, 0xc054b9e060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc0308c9410}, 0xc054b9e000, {{{{0x0, 0x0}, {0xc090c0afc0, 0x2c}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0308c9410}, 0x6352410, 0x1baa2f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc0308c9410}, {0xc05007c000, {0x17430110706fcc97, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc0308c9410}, 0xc01773c460, 0xc07c880ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0308c9410}, 0x660089, {0xc05007c000, {0x17430110706fcc97, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0308c9410}, 0xc0613f9a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110706fcc97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110706fcc97, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0308c9140}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0308c9140}, {0xc0109508f0, 0x660089}, 0xc0109508f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0308c9140}, {0xc00102e4e0}, 0xc0613f9900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0308c9050}, 0xc0613f9900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0452ddb30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808494 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0397352c0}, 0xc03592c3f0, 0x1, 0xc00b2fb0a8, 0x519bbc, 0xc083c12840, 0xc066658de0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0397352c0}, 0x120, 0xc00b2fb230, 0x83c127e0, 0x0, 0x660069, 0xc083c12840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0397352c0}, 0xc083c127e0, {{{{0x0, 0x0}, {0xc0ad6398c0, 0x124}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0397352c0}, 0x6352410, 0x32431728) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0397352c0}, {0xc083c126c0, {0x174301105edaad54, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0397352c0}, 0xc02424b900, 0xc00b2fb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0397352c0}, 0x660089, {0xc083c126c0, {0x174301105edaad54, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0397352c0}, 0xc04f7e8b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105edaad54, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105edaad54, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039735200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039735200}, {0xc00b2fd3f8, 0x660089}, 0xc00b2fd400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039735200}, {0x7fe3f1e47538}, 0xc04f7e8900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039735170}, 0xc04f7e8900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}, 0xc05159a6e0, 0xc03592ca68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}, 0x3, 0xc058e09d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b2fd980}}}, {0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}, 0xc05159a6e0, 0xc058e09d00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc039735170}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039735170}, {0x40, 0xc00b2fdb70}, 0xc00b2fdab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc039735170}, {0x4e9d7a0, 0xc04f7e8900}, 0xc05159a6e0, 0xc03592ca68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc039735170}, 0xc023be45a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc083c125a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc083c125a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807326 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc060f982d0, {0x63c85b8, 0xc075526de0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc075526de0}, {0x4f67ad2, 0x6446ff8}, 0xc002c9cfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219795469 [runnable]: sync.runtime_SemacquireMutex(0xc08b4de690, 0x8f, 0x7fe2d358ca80) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc08b4de538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc08b4de560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc04c44ad80, {0xffffffffffffff15, 0xc08b4de6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc04c44ad80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc04c44ad80, {0xc00d53f980, 0x32, 0x32}, {0xc022269800, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0b1eda398, {0xc00d53f980, 0x32, 0x32}, {0xc022269800, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0b1ed8cc0, {0xc022269800, 0x79, 0x32}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0b1ed8cc0, {0xc00d53f980, 0x79, 0xc022269800}, {0xc022269800, 0xc0820899e0, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0b1ed8a00, {0xc022269800, 0x32, 0x79}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0448403a8, {{0xc0820899e0, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc064395400, {{0xc04a933860, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04e059c00, {0x63c85b8, 0xc055ff6ab0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc055ff6ab0}, {0x64a6560, 0xc064395400}, {0xc04a933860, 0x65, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc055ff6ab0}, {0x6466ed8, 0xc044840100}, {0xc04a933860, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02f930bd0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0521a0600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0200d5000, {0x63c85b8, 0xc055ff6ab0}, 0xc0521a0600, 0xc06891b0e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0200d5000, {0x63c85b8, 0xc055ff6ab0}, 0xc0521a0600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc055ff6a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc055ff6a20}, {0xc0032fe8f0, 0x660089}, 0xc0032fe8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc055ff6a20}, {0xc00102e4e0}, 0xc0521a0500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc055ff69c0}, 0xc0521a0500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc062d70c30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219699904 [IO wait]: internal/poll.runtime_pollWait(0x7fe422a596d0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc0d405b500, 0xc06e3de000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0d405b500, {0xc06e3de000, 0x2b7d, 0x2b7d}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0d405b500, {0xc06e3de000, 0x2b7d, 0xc05f5d48c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0634afcf8, {0xc06e3de000, 0x7eda79, 0xc06e3de005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc07192cea0, {0xc06e3de000, 0x0, 0x2b7d}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04033e6f0, {0xc06e3de000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc096805af8, {0x631b900, 0xc04033e6f0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc096805880, {0x7fe42ab59310, 0xc07192cea0}, 0x2b7d) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc096805880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc096805880, {0xc0af72e000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0246f82d0, {0xc0af72e000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0607cd588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0607cd588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0607cdb60, {0x634aac0, 0xc0607cd588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0607cd500, 0xc056004340, 0xc0750e342b, 0xc0750e342d, {0x635bec0, 0xc03a77ac60}, 0xc000d6ae00, 0xc06d5f1bb8, 0xc0750e3429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0607cd500, {0x63c85b8, 0xc05c0080c0}, 0xc056004330, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc098536037, 0x8}}, 0x0, 0xc05c008030, 0xc05c008060, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc05c0080c0}, {0x643d810, 0xc07192cea0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc03ba88ea0}, {0x643d810, 0xc07192cea0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc03ba88ea0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812672 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801007 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01ffde800, {0x63c85b8, 0xc0865fa930}, 0xc041afda00, 0xc064d9e8c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ffde800, {0x63c85b8, 0xc0865fa930}, 0xc041afda00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011018be2cfd, 0x0, 0x0}, 0xc06410cc90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011018be2cfd, 0x0, 0x0}, 0xc06410cc90, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0865fa8a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0865fa8a0}, {0xc0766d3918, 0x660089}, 0xc0766d3920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0865fa8a0}, {0xc00102e4e0}, 0xc041afd900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0865fa840}, 0xc041afd900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06bc5e1e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0865fa7b0}, {0xc00dfd36b0, 0x24}, 0x46c7c1a16, 0xc00776ed40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0xc09d032270, 0xc00e7a0c78}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0865fa7b0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219747867 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc05b97a580, {0x63c85b8, 0xc01c2f11d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc099ff9300, {0x63c85b8, 0xc01c2f11d0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc099ff9300, {0x63c8510, 0xc03ee79680}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc099ff9300, {0x63c8510, 0xc03ee79680}, {0x63f7db8, 0x9bf6f48}, 0xc0857f7088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc099ff9300, {0x63c85b8, 0xc01e893ef0}, {{0x63f7db8, 0x9bf6f48}, {0xc07845f6ec, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc099ff9300, {0x63c85b8, 0xc01e893ef0}, {{0x63f7db8, 0x9bf6f48}, {0xc07845f6ec, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc07845f6ec, 0x6}, 0x0, 0x0}, {0x24b5740b, 0xedb7a8350, 0x0}, {0x24b57e0e, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc099ff9300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc099ff9300, {0x63c8510, 0xc04708f580}, 0xc06f207dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03f276560) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc04708f580}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03f276560) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802250 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc05655b380}, 0xc044338288, 0x1, 0x0, 0x1, 0xc0c0f24720, 0xc03213e2a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05655b380}, 0x120, 0xc0615d2728, 0xc0f246c0, 0x0, 0x660069, 0xc0c0f24720, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc05655b380}, 0xc0c0f246c0, {{{{0x0, 0x0}, {0xc0141e2000, 0xb9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05655b380}, 0x6352410, 0x5ae3cd58) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc05655b380}, {0xc05f3318c0, {0x1743011020b132de, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc05655b380}, 0xc082adb9a0, 0xc0768eeee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05655b380}, 0x660089, {0xc05f3318c0, {0x1743011020b132de, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc05655b380}, 0xc0a4815500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020b132de, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020b132de, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05655b2f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05655b2f0}, {0xc0d68328f0, 0x660089}, 0xc0d68328f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05655b2f0}, {0xc00102e4e0}, 0xc0a4815400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05655b290}, 0xc0a4815400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc05655a570}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802143 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc06ffeec00, {0x63c85b8, 0xc041c9b4a0}, 0xc062531b00, 0xc07401f180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc06ffeec00, {0x63c85b8, 0xc041c9b4a0}, 0xc062531b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011020584514, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011020584514, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc041c9b410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041c9b410}, {0xc0573a73f8, 0x660089}, 0xc0573a7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc041c9b410}, {0x7fe42930c900}, 0xc062531900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc041c9b320}, 0xc062531900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}, 0xc039a9eea0, 0xc06ba7b3e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}, 0x3, 0xc03abe7f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0573a7980}}}, {0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}, 0xc039a9eea0, 0xc03abe7f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc041c9b320}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc041c9b320}, {0x40, 0xc0573a7b70}, 0xc0573a7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc041c9b320}, {0x4e9d7a0, 0xc062531900}, 0xc039a9eea0, 0xc06ba7b3e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc041c9b320}, 0xc050a7ec60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc041b786c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc041b786c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804966 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016f5da40, {0x63c85b8, 0xc04ef997a0}, 0xc0299e34d0, 0x1, 0xc079fa70a8, 0x519bbc, 0xc06eaf22a0, 0xc06f602060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04ef997a0}, 0x120, 0xc079fa7230, 0x6eaf2240, 0xc0a09704e0, 0x18, 0xc06eaf22a0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016f5da40, {0x63c85b8, 0xc04ef997a0}, 0xc06eaf2240, {{{{0x0, 0x0}, {0xc0604925a0, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ef997a0}, 0x6352410, 0x421b1850) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016fda500, {0x63c85b8, 0xc04ef997a0}, {0xc06eaf2120, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016fda500, {0x63c85b8, 0xc04ef997a0}, 0xc0509350e0, 0xc079fa79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ef997a0}, 0x660089, {0xc06eaf2120, {0x174301103b49e9d3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc017050000, {0x63c85b8, 0xc04ef997a0}, 0xc01aa10800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103b49e9d3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ef99710}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef99710}, {0xc079fa93f8, 0x660089}, 0xc079fa9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ef99710}, {0xc0015221e0}, 0xc01aa10500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ef99650}, 0xc01aa10500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}, 0xc013beb840, 0xc0299e3c08) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}, 0x3, 0xc06f13cd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079fa9980}}}, {0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}, 0xc013beb840, 0xc06f13cd40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ef99650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ef99650}, {0x40, 0xc079fa9b70}, 0xc079fa9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ef99650}, {0x4e9d7a0, 0xc01aa10500}, 0xc013beb840, 0xc0299e3c08) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ef99650}, 0xc041535680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06eaf2000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06eaf2000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812520 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803246 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01e28ec00, {0x63c85b8, 0xc06c8a8690}, 0xc08c08dd00, 0xc06ada7ea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01e28ec00, {0x63c85b8, 0xc06c8a8690}, 0xc08c08dd00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ed6dda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ed6dda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06c8a8450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06c8a8450}, {0xc090c2e7f8, 0x660089}, 0xc090c2e800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06c8a8450}, {0xc00102e4e0}, 0xc08c08dc00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06c8a80f0}, 0xc08c08dc00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc04fc5d970, {0x63c85b8, 0xc0112cde00}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc06421d900, {0x63c85b8, 0xc0112cde00}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc06421d900, {0x63c8510, 0xc0151465c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc06421d900, {0x63c8510, 0xc0151465c0}, {0x63f7db8, 0x9bf6f48}, 0xc090c33088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc06421d900, {0x63c85b8, 0xc03ecb0120}, {{0x63f7db8, 0x9bf6f48}, {0xc0aacd4762, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc06421d900, {0x63c85b8, 0xc03ecb0120}, {{0x63f7db8, 0x9bf6f48}, {0xc0aacd4762, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0aacd4762, 0x6}, 0x0, 0x0}, {0xf9903a3, 0xedb7a8350, 0x0}, {0xf990ae9, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06421d900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06421d900, {0x63c8510, 0xc01e49ec00}, 0xc04c2f1dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc048fce430) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc01e49ec00}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc048fce430) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219803720 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc022befd10}, 0xc0387c6798, 0x1, 0xc0780510a8, 0x519bbc, 0xc040341aa0, 0xc04abde240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc022befd10}, 0x1e0, 0xc078051230, 0x40341a40, 0x0, 0x660069, 0xc040341aa0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc022befd10}, 0xc040341a40, {{{{0x0, 0x0}, {0xc04d732240, 0xef}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc022befd10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc022befd10}, {0xc0590c05a0, {0x174301102d308711, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc022befd10}, 0xc074b84280, 0xc0780519f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc022befd10}, 0x660089, {0xc0590c05a0, {0x174301102d308711, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc022befd10}, 0xc04cffa600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d308711, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d308711, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc022befc80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022befc80}, {0xc0780533f8, 0x660089}, 0xc078053400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc022befc80}, {0xc0015221e0}, 0xc04cffa400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc022befbc0}, 0xc04cffa400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}, 0xc006be1be0, 0xc025fee4c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}, 0x3, 0xc03692fc40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc078053980}}}, {0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}, 0xc006be1be0, 0xc03692fc40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc022befbc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc022befbc0}, {0x40, 0xc078053b70}, 0xc078053ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc022befbc0}, {0x4e9d7a0, 0xc04cffa400}, 0xc006be1be0, 0xc025fee4c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc022befbc0}, 0xc033c106c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590c0480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0590c0480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811043 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02045d2c0, {0x63c85b8, 0xc05205b500}, 0xc02f14c600, 0x1, 0xc010fe10a8, 0x519bbc, 0xc0647d6a00, 0xc0431e1e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06b115b00, {0x63c85b8, 0xc05205b500}, 0xc0647d6a00, 0xc010fe11a8, 0x44, 0x60, 0x0, 0xc0647d6a00, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02045d2c0, {0x63c85b8, 0xc05205b500}, 0xc0647d6700, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc038848a20, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05205b500}, 0x63521a0, 0x3c750160) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02022e4b0, {0x63c85b8, 0xc05205b500}, {0xc0706bbb00, {0x1743011010498bf8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02022e4b0, {0x63c85b8, 0xc05205b500}, 0xc0497f8460, 0xc010fe19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05205b500}, 0x660089, {0xc0706bbb00, {0x1743011010498bf8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020518c00, {0x63c85b8, 0xc05205b500}, 0xc0571d9500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011010498bf8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011010498bf8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05205b470}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05205b470}, {0xc010fe33f8, 0x660089}, 0xc010fe3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05205b470}, {0xc062e0e780}, 0xc0571d9200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05205b350}, 0xc0571d9200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}, 0xc0285403e0, 0xc06ddaa060) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}, 0x3, 0xc01686aec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010fe3980}}}, {0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}, 0xc0285403e0, 0xc01686aec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc05205b350}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05205b350}, {0x40, 0xc010fe3b70}, 0xc010fe3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc05205b350}, {0x4e9d7a0, 0xc0571d9200}, 0xc0285403e0, 0xc06ddaa060) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc05205b350}, 0xc0478aa660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0706bb9e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0706bb9e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806511 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021a5f180, {0x63c85b8, 0xc08b93a480}, 0xc09cb7e120, 0x1, 0xc00d145670, 0x519bbc, 0xc0a6525740, 0xc071b489c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc060030a20, {0x63c85b8, 0xc08b93a480}, 0xc0a6525740, 0xc00d145770, 0x30, 0x40, 0x0, 0xc0a6525740, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021a5f180, {0x63c85b8, 0xc08b93a480}, 0xc0a6525500, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0784b7320, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08b93a480}, 0x63521a0, 0x17e2f798) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a30c80, {0x63c85b8, 0xc08b93a480}, {0xc064e9a360, {0x1743011002b3433b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a30c80, {0x63c85b8, 0xc08b93a480}, 0xc0467994a0, 0xc00d145fb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08b93a480}, 0x660089, {0xc064e9a360, {0x1743011002b3433b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0218b0400, {0x63c85b8, 0xc08b93a480}, 0xc085d7e200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011002b3433b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011002b3433b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08b93a3f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08b93a3f0}, {0xc089d559c0, 0x660089}, 0xc089d559c8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08b93a3f0}, {0xc00102e4e0}, 0xc085d7e100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08b93a390}, 0xc085d7e100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x63c85b8, 0xc08b93a1e0}, {0x7fe422a51230, 0xc07d646a80}, 0xc021bb8360, {0xc086662510, 0x3, 0x3}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x24d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x63c85b8, 0xc08b93a1e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809257 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ba81e00, {0x63c85b8, 0xc0720f4b70}, 0xc045221680, 0x1, 0xc0066870a8, 0x519bbc, 0xc06d0ab920, 0xc053008840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0720f4b70}, 0x120, 0xc006687230, 0x6d0ab8c0, 0x0, 0x660069, 0xc06d0ab920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ba81e00, {0x63c85b8, 0xc0720f4b70}, 0xc06d0ab8c0, {{{{0x0, 0x0}, {0xc0667e8ea0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0720f4b70}, 0x6352410, 0x603b9240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02bae0230, {0x63c85b8, 0xc0720f4b70}, {0xc06d0ab7a0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02bae0230, {0x63c85b8, 0xc0720f4b70}, 0xc04c6fec80, 0xc0066879f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0720f4b70}, 0x660089, {0xc06d0ab7a0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02badbc00, {0x63c85b8, 0xc0720f4b70}, 0xc04fe3d600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0720f4a50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0720f4a50}, {0xc0066893f8, 0x660089}, 0xc006689400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0720f4a50}, {0x0}, 0xc04fe3d400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0720f49c0}, 0xc04fe3d400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}, 0xc07c184380, 0xc0430a4198) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}, 0x3, 0xc06157a0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006689980}}}, {0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}, 0xc07c184380, 0xc06157a0c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0720f49c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0720f49c0}, {0x40, 0xc006689b70}, 0xc006689ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0720f49c0}, {0x4e9d7a0, 0xc04fe3d400}, 0xc07c184380, 0xc0430a4198) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0720f49c0}, 0xc074f9b620, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06d0ab680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc06d0ab680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805134 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ba0f900, {0x63c85b8, 0xc05365b6e0}, 0xc0227386f0, 0x1, 0x0, 0x1, 0xc072d51380, 0xc04d7a9c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc05365b6e0}, 0x120, 0xc06d93a728, 0x72d51320, 0xc0a0971ba0, 0x18, 0xc072d51380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ba0f900, {0x63c85b8, 0xc05365b6e0}, 0xc072d51320, {{{{0x0, 0x0}, {0xc084814120, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc05365b6e0}, 0x6352410, 0x71c3ea10) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b9f8ff0, {0x63c85b8, 0xc05365b6e0}, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b9f8ff0, {0x63c85b8, 0xc05365b6e0}, 0xc0cee0b9a0, 0xc076674ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc05365b6e0}, 0x660089, {0xc0721f59e0, {0x174301102cc21abf, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ba64000, {0x63c85b8, 0xc05365b6e0}, 0xc063d41600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc21abf, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05365b5f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05365b5f0}, {0xc09359a8f0, 0x660089}, 0xc09359a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05365b5f0}, {0xc00102e4e0}, 0xc063d41500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05365b590}, 0xc063d41500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02a7291d0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806769 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc09074ea80}, 0xc01cb7e558, 0x1, 0x708017eeff7f7f, 0x37e11d600, 0xc06fac3800, 0xc06617b4a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc09074ea80}, 0x120, 0xc00345f230, 0x6fac37a0, 0x0, 0x660069, 0xc06fac3800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc09074ea80}, 0xc06fac37a0, {{{{0x0, 0x0}, {0xc083859b00, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc09074ea80}, 0x6352410, 0x23d18c50) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc09074ea80}, {0xc06fac3680, {0x174301104de41b45, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc09074ea80}, 0xc0670240a0, 0xc00345f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc09074ea80}, 0x660089, {0xc06fac3680, {0x174301104de41b45, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc09074ea80}, 0xc0981f2000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104de41b45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104de41b45, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09074e9f0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09074e9f0}, {0xc0034613f8, 0x660089}, 0xc003461400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09074e9f0}, {0x3feffe1328e8ab1a}, 0xc097d25e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09074e960}, 0xc097d25e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}, 0xc09097eba0, 0xc01cb7e528) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}, 0x3, 0xc08cfa3b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003461980}}}, {0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}, 0xc09097eba0, 0xc08cfa3b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09074e960}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09074e960}, {0x40, 0xc003461b70}, 0xc003461ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09074e960}, {0x4e9d7a0, 0xc097d25e00}, 0xc09097eba0, 0xc01cb7e528) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09074e960}, 0xc05c888ea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06fac3560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc06fac3560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811047 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc03bbbaa20}, 0xc04aaf17e8, 0x1, 0xc0229d30a8, 0x519bbc, 0xc0606056e0, 0xc06bc6c060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03bbbaa20}, 0x120, 0xc0229d3230, 0x60605680, 0x0, 0x660069, 0xc0606056e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc03bbbaa20}, 0xc060605680, {{{{0x0, 0x0}, {0xc0a8053440, 0x103}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03bbbaa20}, 0x6352410, 0x50b35dc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc03bbbaa20}, {0xc060605560, {0x174301106bf9e35a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc03bbbaa20}, 0xc083df4be0, 0xc0229d39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03bbbaa20}, 0x660089, {0xc060605560, {0x174301106bf9e35a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc03bbbaa20}, 0xc07c52c900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bf9e35a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bf9e35a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03bbba930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bbba930}, {0xc0229d53f8, 0x660089}, 0xc0229d5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03bbba930}, {0x0}, 0xc07c52c700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03bbba8a0}, 0xc07c52c700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}, 0xc01fec91a0, 0xc08324b5d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}, 0x3, 0xc041d63380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0229d5980}}}, {0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}, 0xc01fec91a0, 0xc041d63380) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03bbba8a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bbba8a0}, {0x40, 0xc0229d5b70}, 0xc0229d5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03bbba8a0}, {0x4e9d7a0, 0xc07c52c700}, 0xc01fec91a0, 0xc08324b5d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03bbba8a0}, 0xc04fcf9ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc000f69320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc000f69320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807438 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0385d79e0}, 0xc025da1950, 0x1, 0xc0075e90a8, 0x519bbc, 0xc0831976e0, 0xc06b5c3320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0385d79e0}, 0x1e0, 0xc0075e9230, 0x83197680, 0x0, 0x660069, 0xc0831976e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0385d79e0}, 0xc083197680, {{{{0x0, 0x0}, {0xc09ed7f440, 0x119}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x649b7a8, {0x63c85b8, 0xc0385d79e0}, 0x41140a5ac7a646ab, 0x580f26af) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0385d79e0}, {0xc01c5b87e0, {0x174301105423916c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0385d79e0}, 0xc060d292c0, 0xc0075e99f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0385d79e0}, 0x660089, {0xc01c5b87e0, {0x174301105423916c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0385d79e0}, 0xc099a19800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105423916c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105423916c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0385d7950}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0385d7950}, {0xc0075eb3f8, 0x660089}, 0xc0075eb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0385d7950}, {0x0}, 0xc099a19600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0385d78c0}, 0xc099a19600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}, 0xc089406ce0, 0xc030be1518) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}, 0x3, 0xc08545f680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0075eb980}}}, {0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}, 0xc089406ce0, 0xc08545f680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0385d78c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0385d78c0}, {0x40, 0xc0075ebb70}, 0xc0075ebab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0385d78c0}, {0x4e9d7a0, 0xc099a19600}, 0xc089406ce0, 0xc030be1518) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0385d78c0}, 0xc02b742540, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01c5b8120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01c5b8120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811048 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01f9be000, {0x63c85b8, 0xc038c4bb90}, 0xc015145c80, 0x1, 0xc0146450a8, 0x519bbc, 0xc03fb3fda0, 0xc01143e840) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc038c4bb90}, 0x120, 0xc014645230, 0x3fb3fd40, 0x0, 0x660069, 0xc03fb3fda0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01f9be000, {0x63c85b8, 0xc038c4bb90}, 0xc03fb3fd40, {{{{0x0, 0x0}, {0xc0416bab40, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc038c4bb90}, 0x6352410, 0x51b89fc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01f609c70, {0x63c85b8, 0xc038c4bb90}, {0xc03fb3fc20, {0x174301106bf9e35a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01f609c70, {0x63c85b8, 0xc038c4bb90}, 0xc04c753860, 0xc0146459f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc038c4bb90}, 0x660089, {0xc03fb3fc20, {0x174301106bf9e35a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f82e800, {0x63c85b8, 0xc038c4bb90}, 0xc08b2e7300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bf9e35a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bf9e35a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc038c4bb00}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc038c4bb00}, {0xc0146473f8, 0x660089}, 0xc014647400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc038c4bb00}, {0x0}, 0xc08b2e7100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc038c4ba70}, 0xc08b2e7100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}, 0xc03e8a1920, 0xc03a288708) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}, 0x3, 0xc01f41c340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc014647980}}}, {0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}, 0xc03e8a1920, 0xc01f41c340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc038c4ba70}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc038c4ba70}, {0x40, 0xc014647b70}, 0xc014647ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc038c4ba70}, {0x4e9d7a0, 0xc08b2e7100}, 0xc03e8a1920, 0xc03a288708) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc038c4ba70}, 0xc0359065a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc000f69560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc000f69560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807439 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc020f888c0, {0x63c85b8, 0xc0385d7830}, 0xc040547950, 0x1, 0xc0074330a8, 0x519bbc, 0xc083197500, 0xc0489c9c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0385d7830}, 0x1e0, 0xc007433230, 0x831974a0, 0x0, 0x660069, 0xc083197500, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc020f888c0, {0x63c85b8, 0xc0385d7830}, 0xc0831974a0, {{{{0x0, 0x0}, {0xc046c1d8c0, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x649b418, {0x63c85b8, 0xc0385d7830}, 0x3f71594949f263fd, 0x48000000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0211060f0, {0x63c85b8, 0xc0385d7830}, {0xc01c5b85a0, {0x174301105423916c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0211060f0, {0x63c85b8, 0xc0385d7830}, 0xc0769c9400, 0xc0074339f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0385d7830}, 0x660089, {0xc01c5b85a0, {0x174301105423916c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c0400, {0x63c85b8, 0xc0385d7830}, 0xc099a19500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105423916c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105423916c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0385d77a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0385d77a0}, {0xc0074353f8, 0x660089}, 0xc007435400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0385d77a0}, {0x3fefffff7865836b}, 0xc099a19300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0385d7710}, 0xc099a19300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}, 0xc089406c60, 0xc030be14b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}, 0x3, 0xc08545f600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007435980}}}, {0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}, 0xc089406c60, 0xc08545f600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0385d7710}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0385d7710}, {0x40, 0xc007435b70}, 0xc007435ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0385d7710}, {0x4e9d7a0, 0xc099a19300}, 0xc089406c60, 0xc030be14b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0385d7710}, 0xc02b7423c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01c5b8360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01c5b8360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801567 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc039a72240}, 0xc0368d1d10, 0x1, 0xc055481ff0, 0x519bbc, 0xc0365aa240, 0xc035a2e240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc039a72240}, 0x1e0, 0xc055482178, 0x365aa1e0, 0x0, 0x660069, 0xc0365aa240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc039a72240}, 0xc0365aa1e0, {{{{0x0, 0x0}, {0xc04f93eb40, 0x46}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc039a72240}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc039a72240}, {0xc02a6bad80, {0x174301101cb07048, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc039a72240}, 0xc0854125a0, 0xc0966fe938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc039a72240}, 0x660089, {0xc02a6bad80, {0x174301101cb07048, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc039a72240}, 0xc049f1ea00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101cb07048, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101cb07048, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039a721b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039a721b0}, {0xc096700340, 0x660089}, 0xc096700348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039a721b0}, {0xc00102e4e0}, 0xc049f1e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039a72150}, 0xc049f1e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc0399003c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219490984 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc067cc02d0, 0x12c) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc0b5310000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc0b5310660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03ebe2600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03ebe2600, {0x63c8510, 0xc067cc0340}, 0xc095fbfdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02b7b1590) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc067cc0340}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc02b7b1590) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801217 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc02349a210}, 0xc0446169a8, 0x1, 0xc05040f0a8, 0x519bbc, 0xc033574f60, 0xc028e8aba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02349a210}, 0x1e0, 0xc05040f230, 0x33574f00, 0x0, 0x660069, 0xc033574f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc02349a210}, 0xc033574f00, {{{{0x0, 0x0}, {0xc040364fc0, 0x3d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x2c3830393333382e, {0x63c85b8, 0xc02349a210}, 0x3134343638302e31, 0x7b202c7d) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc02349a210}, {0xc07afedc20, {0x17430110150fcf76, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc02349a210}, 0xc04e1d6820, 0xc05040f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02349a210}, 0x660089, {0xc07afedc20, {0x17430110150fcf76, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc02349a210}, 0xc01338f400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110150fcf76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110150fcf76, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02349a180}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02349a180}, {0xc0504113f8, 0x660089}, 0xc050411400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02349a180}, {0x726f6f4378227b20}, 0xc01338f200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02349a0f0}, 0xc01338f200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}, 0xc013022f60, 0xc01d2a44c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}, 0x3, 0xc06373ad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc050411980}}}, {0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}, 0xc013022f60, 0xc06373ad40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02349a0f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02349a0f0}, {0x40, 0xc050411b70}, 0xc050411ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02349a0f0}, {0x4e9d7a0, 0xc01338f200}, 0xc013022f60, 0xc01d2a44c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02349a0f0}, 0xc0286bfa40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05f61cc60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05f61cc60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810279 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc0429af290}, 0xc05f8ab600, 0xc07aaaa320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0429af290}, 0xc05f8ab600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c880269, 0x0, 0x0}, 0xc03399f740, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c880269, 0x0, 0x0}, 0xc03399f740, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0429af200}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0429af200}, {0xc010d6f3f8, 0x660089}, 0xc010d6f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0429af200}, {0xc01b5bc820}, 0xc05f8ab400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0429af170}, 0xc05f8ab400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}, 0xc00d92b600, 0xc028acf950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}, 0x3, 0xc043b8cf40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc010d6f980}}}, {0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}, 0xc00d92b600, 0xc043b8cf40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0429af170}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0429af170}, {0x40, 0xc010d6fb70}, 0xc010d6fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0429af170}, {0x4e9d7a0, 0xc05f8ab400}, 0xc00d92b600, 0xc028acf950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0429af170}, 0xc01bb88c00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0638b07e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0638b07e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219804584 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc02bf32db0}, 0xc00065b530, 0x1, 0xc052ef70a8, 0x519bbc, 0xc051696d80, 0xc04f049e60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02bf32db0}, 0x1e0, 0xc052ef7230, 0x51696d20, 0x0, 0x660069, 0xc051696d80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc02bf32db0}, 0xc051696d20, {{{{0x0, 0x0}, {0xc07c2d3b00, 0x54}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02bf32db0}, 0x63521a0, 0x3d0ac298) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc02bf32db0}, {0xc063f04c60, {0x17430110356c23ac, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02bf32db0}, 0xc046885680, 0xc052ef79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02bf32db0}, 0x660089, {0xc063f04c60, {0x17430110356c23ac, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02bf32db0}, 0xc076d2c600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110356c23ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110356c23ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02bf32c30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02bf32c30}, {0xc052ef93f8, 0x660089}, 0xc052ef9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02bf32c30}, {0xc0015221e0}, 0xc076d2c400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02bf32750}, 0xc076d2c400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}, 0xc0994b4be0, 0xc02e7120d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}, 0x3, 0xc050a928c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc052ef9980}}}, {0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}, 0xc0994b4be0, 0xc050a928c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02bf32750}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02bf32750}, {0x40, 0xc052ef9b70}, 0xc052ef9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02bf32750}, {0x4e9d7a0, 0xc076d2c400}, 0xc0994b4be0, 0xc02e7120d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02bf32750}, 0xc028848480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc063f04b40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc063f04b40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810262 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01d898800, {0x63c85b8, 0xc02dfbe480}, 0xc059954d00, 0xc02e47c820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc02dfbe480}, 0xc059954d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c3cf546, 0x0, 0x0}, 0xc03cc76a40, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c3cf546, 0x0, 0x0}, 0xc03cc76a40, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02dfbe300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02dfbe300}, {0xc0246f33f8, 0x660089}, 0xc0246f3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02dfbe300}, {0x726f6674616c706e}, 0xc059954900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02dfbe180}, 0xc059954900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}, 0xc00f3f4200, 0xc03135a438) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}, 0x3, 0xc063d6a980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0246f3980}}}, {0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}, 0xc00f3f4200, 0xc063d6a980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02dfbe180}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02dfbe180}, {0x40, 0xc0246f3b70}, 0xc0246f3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02dfbe180}, {0x4e9d7a0, 0xc059954900}, 0xc00f3f4200, 0xc03135a438) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02dfbe180}, 0xc0aadfbc80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06fba06c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc06fba06c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800136 [runnable]: sync.runtime_SemacquireMutex(0xc095570690, 0x8f, 0xc0955706a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc095570538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc095570560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc036aded80, {0xffffffffffffff15, 0xc0955706c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc036aded80) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc036aded80, {0xc04194e500, 0x32, 0xa0}, {0xc06d6d6880, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09347d998, {0xc04194e500, 0x32, 0xa0}, {0xc06d6d6880, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09347c2c0, {0xc06d6d6880, 0x77, 0xa0}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09347c2c0, {0xc04194e500, 0x77, 0xc06d6d6880}, {0xc06d6d6880, 0xc05b3eae10, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09347c000, {0xc06d6d6880, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc05a439e28, {{0xc05b3eae10, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0224a0300, {{0xc04a932090, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc079a9f800, {0x63c85b8, 0xc0254af440}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0254af440}, {0x64a6560, 0xc0224a0300}, {0xc04a932090, 0x21036, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0254af440}, {0x6466ed8, 0xc05a439b80}, {0xc04a932090, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc01b19fb60}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05f9f1600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a31800, {0x63c85b8, 0xc0254af440}, 0xc05f9f1600, 0xc0877cff40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a31800, {0x63c85b8, 0xc0254af440}, 0xc05f9f1600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0254af350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0254af350}, {0xc0031068f0, 0x660089}, 0xc0031068f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0254af350}, {0xc00102e4e0}, 0xc05f9f1500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0254af1d0}, 0xc05f9f1500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d350}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807875 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc06f0135c0}, 0xc026d9f518, 0x1, 0xc00e7c50a8, 0x519bbc, 0xc00e868540, 0xc03c574540) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01f066a80, {0x63c85b8, 0xc06f0135c0}, 0xc059d77d40, 0xc00e7c51a8, 0xe868540, 0x0, 0x0, 0xc00e868540, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc06f0135c0}, 0xc00e868480, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc047f4cb40, 0x65}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06f0135c0}, 0x6352260, 0x3b957448) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc06f0135c0}, {0xc00e868120, {0x174301101e0e3064, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc06f0135c0}, 0xc02f05c8c0, 0xc00e7c59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06f0135c0}, 0x660089, {0xc00e868120, {0x174301101e0e3064, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc06f0135c0}, 0xc06daa0400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101e0e3064, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101e0e3064, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06f013530}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f013530}, {0xc00e7c73f8, 0x660089}, 0xc00e7c7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06f013530}, {0x3d7756454c4c6c33}, 0xc06daa0200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06f0134a0}, 0xc06daa0200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}, 0xc02173f9a0, 0xc0aad7eeb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}, 0x3, 0xc0392f7340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e7c7980}}}, {0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}, 0xc02173f9a0, 0xc0392f7340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06f0134a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06f0134a0}, {0x40, 0xc00e7c7b70}, 0xc00e7c7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06f0134a0}, {0x4e9d7a0, 0xc06daa0200}, 0xc02173f9a0, 0xc0aad7eeb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06f0134a0}, 0xc06c0e5a40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00e868000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc00e868000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809531 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c0cf540, {0x63c85b8, 0xc08826f9b0}, 0xc0249e14a0, 0x1, 0xc01c9210a8, 0x519bbc, 0xc086f69920, 0xc039a6e960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08826f9b0}, 0x120, 0xc01c921230, 0x86f698c0, 0x0, 0x660069, 0xc086f69920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c0cf540, {0x63c85b8, 0xc08826f9b0}, 0xc086f698c0, {{{{0x0, 0x0}, {0xc014d9b7a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08826f9b0}, 0x6352410, 0x3102edf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c0d49b0, {0x63c85b8, 0xc08826f9b0}, {0xc086f697a0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c0d49b0, {0x63c85b8, 0xc08826f9b0}, 0xc0d0452b40, 0xc01c9219f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08826f9b0}, 0x660089, {0xc086f697a0, {0x174301105e935103, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bd95800, {0x63c85b8, 0xc08826f9b0}, 0xc05b358c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105e935103, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08826f920}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08826f920}, {0xc01c9233f8, 0x660089}, 0xc01c923400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08826f920}, {0x40e427ffce0760fc}, 0xc05b358a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08826f890}, 0xc05b358a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}, 0xc024415960, 0xc034dc3f68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}, 0x3, 0xc0af72cbc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01c923980}}}, {0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}, 0xc024415960, 0xc0af72cbc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08826f890}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08826f890}, {0x40, 0xc01c923b70}, 0xc01c923ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08826f890}, {0x4e9d7a0, 0xc05b358a00}, 0xc024415960, 0xc034dc3f68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08826f890}, 0xc02f4cca20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086f69680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc086f69680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805344 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc04f1a7830}, 0xc015219350, 0x1, 0x708017dfd890e3, 0x37e11d600, 0xc05b3cd6e0, 0xc0679454a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04f1a7830}, 0x120, 0xc07bf5b230, 0x5b3cd680, 0xc0431b8000, 0x18, 0xc05b3cd6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc04f1a7830}, 0xc05b3cd680, {{{{0x0, 0x0}, {0xc055a9b200, 0x21}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04f1a7830}, 0x6352410, 0x24ff0fc8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc04f1a7830}, {0xc05b3cd560, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc04f1a7830}, 0xc0bb549720, 0xc07bf5b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04f1a7830}, 0x660089, {0xc05b3cd560, {0x174301102fc7ab04, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc04f1a7830}, 0xc052a72300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fc7ab04, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04f1a77a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f1a77a0}, {0xc07bf5d3f8, 0x660089}, 0xc07bf5d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04f1a77a0}, {0x4d939c}, 0xc052a72100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04f1a7710}, 0xc052a72100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}, 0xc0656ebea0, 0xc015219308) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}, 0x3, 0xc06c770f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07bf5d980}}}, {0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}, 0xc0656ebea0, 0xc06c770f80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04f1a7710}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04f1a7710}, {0x40, 0xc07bf5db70}, 0xc07bf5dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}, 0x20301b, 0x20301b) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04f1a7710}, {0x4e9d7a0, 0xc052a72100}, 0xc0656ebea0, 0xc015219308) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04f1a7710}, 0xc0ce587200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b776240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05b776240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219675234 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc06f8526c0) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc0cd4ff3f0, {0x4d83cc0, 0xc06481af00}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x1f3) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc06f852480, 0xc08a3dfe30, 0xc08a3dfe00) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc06f852480, {0x4d83cc0, 0xc06481af00}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc0909a9ef0}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc050f94700}, {0x4d83cc0, 0xc06481af00}, 0x2, {0xc012856840, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc0909a9ef0}, {0x4f6406d, 0x7fe457d7ff18}, {0x4e9d7a0, 0xc050f94700}, {0x4d83cc0, 0xc06481af00}, 0x53db1a13542aafaf, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x7b949dd25faa69d4, {0x63c85b8, 0xc0909a9ef0}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc050f94700}, {0x4d83cc0, 0xc06481af00}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc033a92ed8, {0x63c85b8, 0xc0909a9ef0}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc033a92ed8}}, {0x63c85b8, 0xc0909a9ef0}, 0xc050f94700, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1c, 0x1c, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc0a018b0e0, {0x63c85b8, 0xc051c56a50}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc0a018b0e0, {0x63c85b8, 0xc051c56a50}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc09099cbe0, {0x63c85b8, 0xc051c56a50}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).nextKey(0xc049af3400, {0x63c85b8, 0xc051c56a50}) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:539 +0x87 github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).StartScanFrom(0xc049af3400, {0x63c85b8, 0xc051c56a50}, {0x6344c08, 0xc0a018b0e0}, 0x3d) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:509 +0x108 github.com/cockroachdb/cockroach/pkg/sql/row.(*Fetcher).StartScan(0xc049af3400, {0x63c85b8, 0xc051c56a50}, 0xc00c82d970, {0xc090f80000, 0x1, 0x400}, 0x4f0b32c, 0x0, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/row/fetcher.go:379 +0x32a github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc0151f8000) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:902 +0xaf8 github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc0151f8000) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:666 +0x70 github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc0151f9800) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:747 +0x22d github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc0151f9800) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:666 +0x70 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc055936c00) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:210 +0x130 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*allSpooler).spool(0xc0ce587d40) github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:136 +0x37 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*sortOp).Next(0xc05f04d2c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:280 +0x7a github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1() github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:198 +0x2f github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x7fe457d7f5b8) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc040778480) github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:196 +0x68 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*fnOp).Next(0xc015ce6000) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/fn_op.go:32 +0x29 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*distinctBytesOp).Next(0xc0ce587f20) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/colexecbase/distinct.eg.go:471 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc04a8bda90) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:139 +0x2e github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecwindow.(*windowSortingPartitioner).Next(0xc04a8bdae0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecwindow/partitioner.go:68 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc04a8bdb30) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:139 +0x2e github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecwindow.(*rowNumberWithPartitionOp).Next(0xc015ce6040) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/colexecwindow/row_number.eg.go:96 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc02fd382a0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selEQInt64Int64ConstOp).Next(0xc051c56510) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/colexecsel/selection_ops.eg.go:5764 +0x32 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*BatchSchemaSubsetEnforcer).Next(0xc06da6d180) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:206 +0x2e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*bufferOp).advance(...) github.com/cockroachdb/cockroach/pkg/sql/colexec/buffer.go:48 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*caseOp).Next(0xc0365fd860) github.com/cockroachdb/cockroach/pkg/sql/colexec/case.go:193 +0x4a github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc014559800) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colexec.spool_false(0xc046483d40) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/sorttopk.eg.go:310 +0x3e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).spool(0x101010000000068) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/sorttopk.eg.go:67 +0x2c github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).Next(0xc046483d40) github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:165 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1() github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:198 +0x2f github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc076cae220) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc0407785a0) github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:196 +0x68 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc022b41740) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc06f372c00) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc077c9a0b0, {0x63c8510, 0xc029d135c0}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc07dcd8190, {0x63c8510, 0xc029d135c0}, 0xc00c82d970) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc024d4c440}, 0xc00aed2a80, 0xc00c82d970, 0xc0c6946c80, 0xc0cf92ae00, 0xc07990a2e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc024d4c440}, 0xc079909ff0, 0xc00aed2a80, 0xc087b1fbc0, {{0x63ca960, 0xc079d0b220}, 0x0}, 0xc0cf92ae00) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc079909900, {0x63c8510, 0xc024d4c440}, 0xc079909ff0, 0xc024d4c440, {0x7fe42b790cf0, 0xc087b1fbc0}, 0x0, 0xc0207789b8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc079909900, {0x63c8510, 0xc024d4c440}, 0xc079909ff0, {0x7fe42b790cf0, 0xc087b1fbc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc079909900, {0x63c85b8, 0xc059b8fce0}, {{0x63f8d78, 0xc046693590}, {0xc0680ef010, 0x5bf}, 0x1e, 0x4}, 0xc0454f6fc0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc079909900, {0x63c85b8, 0xc059b8fce0}, {{0x63f8d78, 0xc046693590}, {0xc0680ef010, 0x5bf}, 0x1e, 0x4}, 0xc0454f6fc0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc079909900, {0x63c85b8, 0xc059b8fce0}, {0xc0454f6fc0, {0xc071067c20, 0x1e, 0x1e}, {0xc04efedf80, 0x37, 0x37}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0680ef97e, 0x0}, 0x0, {0x9033202, 0xedb7a834d, 0x0}, 0x1}, 0xc079909900, 0xc08a3e7980, 0xc08a3e7970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc079909900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc079909900, {0x63c8510, 0xc06e1d1e80}, 0xc07aa0fdd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc061a4a2b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06e1d1e80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc061a4a2b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812575 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808436 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc0208a5d40}, 0xc0270c33c8, 0x1, 0xc0067abff0, 0x519bbc, 0xc03621c9c0, 0xc058825aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0208a5d40}, 0x1e0, 0xc0067ac178, 0x3621c960, 0x0, 0x660069, 0xc03621c9c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc0208a5d40}, 0xc03621c960, {{{{0x0, 0x0}, {0xc03e794900, 0x42}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0208a5d40}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc0208a5d40}, {0xc05adad680, {0x174301105c9acd56, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc0208a5d40}, 0xc046885040, 0xc076b96938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0208a5d40}, 0x660089, {0xc05adad680, {0x174301105c9acd56, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0208a5d40}, 0xc0c8bda200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c9acd56, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c9acd56, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0208a5cb0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0208a5cb0}, {0xc076b98340, 0x660089}, 0xc076b98348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0208a5cb0}, {0x24}, 0xc0c8bda100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0208a5bf0}, 0xc0c8bda100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc055563a40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219791852 [runnable]: sync.runtime_SemacquireMutex(0xc07d310690, 0x8f, 0x7fdee32e6ec0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc07d310538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc07d310560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc042f36900, {0xffffffffffffff15, 0xc07d3106c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc042f36900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc042f36900, {0xc024e43b80, 0x32, 0x72}, {0xc02cf6fe00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc094e1d998, {0xc024e43b80, 0x32, 0x72}, {0xc02cf6fe00, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc094e1c2c0, {0xc02cf6fe00, 0x74, 0x72}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc094e1c2c0, {0xc024e43b80, 0x74, 0xc02cf6fe00}, {0xc02cf6fe00, 0xc07461d1d0, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc094e1c000, {0xc02cf6fe00, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc00feb6928, {{0xc07461d1d0, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc03b7cf500, {{0xc04756c630, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0bbeafc00, {0x63c85b8, 0xc053ec6cf0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc053ec6cf0}, {0x64a6560, 0xc03b7cf500}, {0xc04756c630, 0x3264565478526c5a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc053ec6cf0}, {0x6466ed8, 0xc00feb6680}, {0xc04756c630, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc039766720}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01bffb200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f82dc00, {0x63c85b8, 0xc053ec6cf0}, 0xc01bffb200, 0xc048082aa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f82dc00, {0x63c85b8, 0xc053ec6cf0}, 0xc01bffb200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053ec6c60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053ec6c60}, {0xc001c988f0, 0x660089}, 0xc001c988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053ec6c60}, {0xc00102e4e0}, 0xc01bffb100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053ec6c00}, 0xc01bffb100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc053ec69c0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804680 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc06d1913e0}, 0xc05776a2a0, 0x1, 0xc0589b30a8, 0x519bbc, 0xc066b8d260, 0xc033c21f20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0457d2060, {0x63c85b8, 0xc06d1913e0}, 0xc05907dc20, 0xc0589b31a8, 0x66b8d2c0, 0x0, 0x1, 0xc066b8d260, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc06d1913e0}, 0xc066b8d200, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc05c3e0240, 0x68}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc06d1913e0}, 0x63523b0, 0x3d0ac6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc06d1913e0}, {0x0, {0x1743011037a69437, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc06d1913e0}, 0xc06c1054a0, 0xc0589b39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06d1913e0}, 0x660089, {0x0, {0x1743011037a69437, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc06d1913e0}, 0xc0701b9200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011037a69437, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011037a69437, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x1743011037a69437, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06d191350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d191350}, {0xc0589b53f8, 0x660089}, 0xc0589b5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06d191350}, {0x0}, 0xc0701b9000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06d1912c0}, 0xc0701b9000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}, 0xc074629cc0, 0xc0501584b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}, 0x3, 0xc05ec88180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0589b5980}}}, {0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}, 0xc074629cc0, 0xc05ec88180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06d1912c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06d1912c0}, {0x40, 0xc0589b5b70}, 0xc0589b5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06d1912c0}, {0x4e9d7a0, 0xc0701b9000}, 0xc074629cc0, 0xc0501584b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06d1912c0}, 0xc0442495c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc066b8d0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc066b8d0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219791666 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc056544720}, 0xc01ad22cb0, {0x646cc48, 0xc0565447e0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc056544720}, 0xedb7a834e, 0xc073772fc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc056544720}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc045a76fc0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01ad22ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc045a76fc0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810893 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018383180, {0x63c85b8, 0xc02c200b70}, 0xc0c884db00, 0x1, 0xc0029df0a8, 0x519bbc, 0xc053e505c0, 0xc05f4b4ae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04560f680, {0x63c85b8, 0xc02c200b70}, 0xc053e505c0, 0xc0029df1a8, 0x30, 0x40, 0x0, 0xc053e505c0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018383180, {0x63c85b8, 0xc02c200b70}, 0xc053e50380, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc0290f9e60, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc02c200b70}, 0x63521a0, 0x120b6760) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0181df9f0, {0x63c85b8, 0xc02c200b70}, {0xc022d206c0, {0x1743010ff7b82cac, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0181df9f0, {0x63c85b8, 0xc02c200b70}, 0xc03e98b400, 0xc0029df9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02c200b70}, 0x660089, {0xc022d206c0, {0x1743010ff7b82cac, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0184f6400, {0x63c85b8, 0xc02c200b70}, 0xc049113000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff7b82cac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff7b82cac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02c200900}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c200900}, {0xc0029e13f8, 0x660089}, 0xc0029e1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02c200900}, {0x40e8faff2ddd4c3a}, 0xc049112e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02c2005d0}, 0xc049112e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}, 0xc0722225a0, 0xc0b27aa0a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}, 0x3, 0xc04fe1c400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0029e1980}}}, {0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}, 0xc0722225a0, 0xc04fe1c400) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02c2005d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02c2005d0}, {0x40, 0xc0029e1b70}, 0xc0029e1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02c2005d0}, {0x4e9d7a0, 0xc049112e00}, 0xc0722225a0, 0xc0b27aa0a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02c2005d0}, 0xc047032b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc022d20480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc022d20480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810225 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc018112500, {0x63c85b8, 0xc0621116b0}, 0xc0ced478a8, 0x1, 0xc01fdc10a8, 0x519bbc, 0xc056dde600, 0xc06e413c00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc01e202000, {0x63c85b8, 0xc0621116b0}, 0xc01e202030, 0xc01fdc11a8, 0x16, 0x18, 0xffffffffffffffff, 0xc056dde600, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc018112500, {0x63c85b8, 0xc0621116b0}, 0xc056dde5a0, {{{{0x0, 0x0}, {0xc0744d9e60, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0621116b0}, 0x6352230, 0x6b2458b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc017e6f450, {0x63c85b8, 0xc0621116b0}, {0xc056dde360, {0x174301106c15575b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc017e6f450, {0x63c85b8, 0xc0621116b0}, 0xc046d595e0, 0xc01fdc19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0621116b0}, 0x660089, {0xc056dde360, {0x174301106c15575b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01811f400, {0x63c85b8, 0xc0621116b0}, 0xc0b8d00000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c15575b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c15575b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062111620}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062111620}, {0xc01fdc33f8, 0x660089}, 0xc01fdc3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062111620}, {0x40e8faff2ddd4c3a}, 0xc02c4f1c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062111590}, 0xc02c4f1c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}, 0xc04ba626c0, 0xc00e5fc0c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}, 0x3, 0xc070da02c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01fdc3980}}}, {0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}, 0xc04ba626c0, 0xc070da02c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062111590}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062111590}, {0x40, 0xc01fdc3b70}, 0xc01fdc3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062111590}, {0x4e9d7a0, 0xc02c4f1c00}, 0xc04ba626c0, 0xc00e5fc0c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062111590}, 0xc038afb8c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc056dde120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc056dde120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801411 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01f61e800, {0x63c85b8, 0xc065a8e150}, 0xc097298d00, 0xc07121b220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f61e800, {0x63c85b8, 0xc065a8e150}, 0xc097298d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101b724a56, 0x0, 0x0}, 0xc00d4f6760, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101b724a56, 0x0, 0x0}, 0xc00d4f6760, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc065a8e0c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc065a8e0c0}, {0xc0766cf918, 0x660089}, 0xc0766cf920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc065a8e0c0}, {0xc00102e4e0}, 0xc097298c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc065a8e060}, 0xc097298c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc03a9b3080}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc0619bdf80}, {0xc00dfd36b0, 0x24}, 0x46c790b8b, 0xc0387f9d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x10, 0xc000510478}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc0619bdf80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804893 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edabb80, {0x63c85b8, 0xc074d45b60}, 0xc019ed4318, 0x1, 0xc07b2790a8, 0x519bbc, 0xc0595dc420, 0xc04d9527e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc074d45b60}, 0x1e0, 0xc07b279230, 0x595dc3c0, 0xc00a953d40, 0x18, 0xc0595dc420, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edabb80, {0x63c85b8, 0xc074d45b60}, 0xc0595dc3c0, {{{{0x0, 0x0}, {0xc09cc32360, 0x4}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc021c573e0, {0x63c85b8, 0xc074d45b60}, 0xc022921530, 0x7b279428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701d60, {0x63c85b8, 0xc074d45b60}, {0xc058edbe60, {0x17430110311c7881, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701d60, {0x63c85b8, 0xc074d45b60}, 0xc01de979a0, 0xc07b2799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc074d45b60}, 0x660089, {0xc058edbe60, {0x17430110311c7881, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01ef44c00, {0x63c85b8, 0xc074d45b60}, 0xc091596300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110311c7881, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110311c7881, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc074d45aa0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074d45aa0}, {0xc07b27b3f8, 0x660089}, 0xc07b27b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc074d45aa0}, {0x0}, 0xc091596100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc074d45a10}, 0xc091596100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}, 0xc01d74ab40, 0xc019ed5950) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}, 0x3, 0xc017b30bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07b27b980}}}, {0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}, 0xc01d74ab40, 0xc017b30bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc074d45a10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc074d45a10}, {0x40, 0xc07b27bb70}, 0xc07b27bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}, 0x203005, 0x203005) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc074d45a10}, {0x4e9d7a0, 0xc091596100}, 0xc01d74ab40, 0xc019ed5950) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc074d45a10}, 0xc029af1b00, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058355560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc058355560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810668 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc04358bf50}, 0xc086aa3600, 0xc02f165400) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04358bf50}, 0xc086aa3600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d6d1d47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d6d1d47, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04358be60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04358be60}, {0xc028f893f8, 0x660089}, 0xc028f89400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04358be60}, {0xc0015221e0}, 0xc086aa3300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04358bdd0}, 0xc086aa3300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}, 0xc019856de0, 0xc02cd81110) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}, 0x3, 0xc04ece7040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc028f89980}}}, {0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}, 0xc019856de0, 0xc04ece7040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04358bdd0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04358bdd0}, {0x40, 0xc028f89b70}, 0xc028f89ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04358bdd0}, {0x4e9d7a0, 0xc086aa3300}, 0xc019856de0, 0xc02cd81110) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04358bdd0}, 0xc06b2378c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0521a6000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc0521a6000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811541 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021ad3040, {0x63c85b8, 0xc01d82a7b0}, 0xc0738dc750, 0x1, 0xc005bc30a8, 0x519bbc, 0xc048f80060, 0xc08cb196e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0681225a0, {0x63c85b8, 0xc01d82a7b0}, 0xc0681225d8, 0xc005bc31a8, 0x30, 0x40, 0xffffffffffffffff, 0xc048f80060, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021ad3040, {0x63c85b8, 0xc01d82a7b0}, 0xc048f80000, {{{{0x0, 0x0}, {0xc065fb4fc0, 0x4}}, {{0xc05b259680, 0x5}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01d82a7b0}, 0x63521a0, 0x54d3a000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021a31b80, {0x63c85b8, 0xc01d82a7b0}, {0xc0589b6120, {0x1743011063805c99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021a31b80, {0x63c85b8, 0xc01d82a7b0}, 0xc058580fa0, 0xc005bc39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01d82a7b0}, 0x660089, {0xc0589b6120, {0x1743011063805c99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c08400, {0x63c85b8, 0xc01d82a7b0}, 0xc0b5c1c200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011063805c99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011063805c99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01d82a720}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d82a720}, {0xc005bc53f8, 0x660089}, 0xc005bc5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01d82a720}, {0x7fe3489f53d0}, 0xc0b5c1c000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01d82a600}, 0xc0b5c1c000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}, 0xc06d764440, 0xc029c0a180) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}, 0x3, 0xc028a440c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005bc5980}}}, {0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}, 0xc06d764440, 0xc028a440c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01d82a600}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d82a600}, {0x40, 0xc005bc5b70}, 0xc005bc5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01d82a600}, {0x4e9d7a0, 0xc0b5c1c000}, 0xc06d764440, 0xc029c0a180) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01d82a600}, 0xc03c8922a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0589b6000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0589b6000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809820 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021c5a280, {0x63c85b8, 0xc07e09b590}, 0xc0292d4660, 0x1, 0x70801804569ade, 0x37e11d600, 0xc081cee600, 0xc07b8abda0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07e09b590}, 0x120, 0xc00e14b230, 0x81cee5a0, 0x0, 0x660069, 0xc081cee600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021c5a280, {0x63c85b8, 0xc07e09b590}, 0xc081cee5a0, {{{{0x0, 0x0}, {0xc033f8e900, 0x3}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07e09b590}, 0x6352410, 0x29b7baf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021c42460, {0x63c85b8, 0xc07e09b590}, {0xc081cee480, {0x1743011067536073, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021c42460, {0x63c85b8, 0xc07e09b590}, 0xc0307a75e0, 0xc00e14b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07e09b590}, 0x660089, {0xc081cee480, {0x1743011067536073, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021c68c00, {0x63c85b8, 0xc07e09b590}, 0xc0bd5b8b00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067536073, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067536073, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07e09b500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e09b500}, {0xc00e14d3f8, 0x660089}, 0xc00e14d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07e09b500}, {0x7fe3f1ee5f60}, 0xc0bd5b8900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07e09b470}, 0xc0bd5b8900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}, 0xc0500312c0, 0xc0292d4630) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}, 0x3, 0xc06fb0c500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00e14d980}}}, {0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}, 0xc0500312c0, 0xc06fb0c500) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07e09b470}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07e09b470}, {0x40, 0xc00e14db70}, 0xc00e14dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07e09b470}, {0x4e9d7a0, 0xc0bd5b8900}, 0xc0500312c0, 0xc0292d4630) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07e09b470}, 0xc0465066c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc081cee360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc081cee360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812254 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc050b47170}, 0xc03e62fa58, 0x1, 0xc0338376d8, 0x519bbc, 0xc02b8ba780, 0xc01341f740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc07a1be900, {0x63c85b8, 0xc050b47170}, 0xc03ecd18c0, 0xc0338377d8, 0x2b8ba780, 0x0, 0x3, 0xc02b8ba780, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc050b47170}, 0xc02b8ba6c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc000583d40, 0xb2}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc050b47170}, 0x6352260, 0x434f49a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc050b47170}, {0xc0110b3440, {0x174301104dbcbff6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc050b47170}, 0xc02eeaabe0, 0xc033838020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc050b47170}, 0x660089, {0xc0110b3440, {0x174301104dbcbff6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc050b47170}, 0xc0c122de00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104dbcbff6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104dbcbff6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e9f5ec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e9f5ec0}, {0xc06f209a28, 0x660089}, 0xc06f209a30) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e9f5ec0}, {0xc00102e4e0}, 0xc077e4c900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01e9f5dd0}, 0xc077e4c900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeat(0xc09c67dd50, {0x63c85b8, 0xc01e9f5d10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:407 +0x44f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc09c67dd50, {0x63c85b8, 0xc01e9f5d10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:343 +0xd3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc01e9f5d10}, {0x4f67ad2, 0x6446ff8}, 0xc01ad24fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219809910 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0122d5800, {0x63c85b8, 0xc04ba9ac30}, 0xc004c40e00, 0xc04c9a9b80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0122d5800, {0x63c85b8, 0xc04ba9ac30}, 0xc004c40e00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110682498e5, 0x0, 0x0}, 0xc031cfd130, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110682498e5, 0x0, 0x0}, 0xc031cfd130, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x14, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ba9aba0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ba9aba0}, {0xc01ca4b3f8, 0x660089}, 0xc01ca4b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ba9aba0}, {0x7fe4293c86e0}, 0xc004c40a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ba9ab10}, 0xc004c40a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}, 0xc056c91b80, 0xc01b3eb7a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}, 0x3, 0xc04c170900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01ca4b980}}}, {0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}, 0xc056c91b80, 0xc04c170900) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ba9ab10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ba9ab10}, {0x40, 0xc01ca4bb70}, 0xc01ca4bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ba9ab10}, {0x4e9d7a0, 0xc004c40a00}, 0xc056c91b80, 0xc01b3eb7a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ba9ab10}, 0xc02989c420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0118c98c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0118c98c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802049 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02e0d4c00, {0x63c85b8, 0xc015b5b770}, 0xc088b64700, 0xc06c204820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc015b5b770}, 0xc088b64700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011024a4c0d9, 0x0, 0x0}, 0xc043634070, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011024a4c0d9, 0x0, 0x0}, 0xc043634070, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc015b5b6e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc015b5b6e0}, {0xc076675a08, 0x660089}, 0xc076675a10) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc015b5b6e0}, {0xc00102e4e0}, 0xc088b64600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc015b5b680}, 0xc088b64600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c85b8, 0xc015b5b5f0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc015b5b5f0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809436 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02b25c500, {0x63c85b8, 0xc01d602690}, 0xc02768c840, 0x1, 0xc01b47f0a8, 0x519bbc, 0xc08a001800, 0xc052b796e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01d602690}, 0x120, 0xc01b47f230, 0x8a0017a0, 0x0, 0x660069, 0xc08a001800, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02b25c500, {0x63c85b8, 0xc01d602690}, 0xc08a0017a0, {{{{0x0, 0x0}, {0xc03e484ea0, 0x17}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01d602690}, 0x6352410, 0x4b1bc940) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02b244500, {0x63c85b8, 0xc01d602690}, {0xc08a001560, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02b244500, {0x63c85b8, 0xc01d602690}, 0xc08953db80, 0xc01b47f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01d602690}, 0x660089, {0xc08a001560, {0x174301105a2a5a99, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02b26c000, {0x63c85b8, 0xc01d602690}, 0xc0600a9600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105a2a5a99, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01d6025d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01d6025d0}, {0xc01b4813f8, 0x660089}, 0xc01b481400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01d6025d0}, {0x7fe4257313b8}, 0xc0600a9400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070383f20}, 0xc0600a9400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}, 0xc0405a3ca0, 0xc03f1806d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}, 0x3, 0xc0be9c2300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01b481980}}}, {0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}, 0xc0405a3ca0, 0xc0be9c2300) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070383f20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070383f20}, {0x40, 0xc01b481b70}, 0xc01b481ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070383f20}, {0x4e9d7a0, 0xc0600a9400}, 0xc0405a3ca0, 0xc03f1806d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070383f20}, 0xc03674bd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08a001440, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc08a001440, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812784 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812570 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219800447 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc09617a548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc09617a5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc07cbcb1f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0d0790380, {0xc0bbb97600, 0x2b, 0x79}, {0xc042a26bd0, 0x2b, 0x6b}, 0x1, 0xb1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc07cbcb088, {0xc0bbb97600, 0x1, 0x6b}, {0xc042a26bd0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07cbcb088, {0xc0bbb97600, 0x2b, 0x79}, {0xc042a26bd0, 0x2b, 0x6b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07cbca7c0, {0xc042a26bd0, 0x6b, 0x79}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07cbca7c0, {0xc0bbb97600, 0x2b, 0xc042a26bd0}, {0xc042a26bd0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07cbca500, {0xc042a26bd0, 0x2b, 0x6b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc05a2d9b88, {{0xc03e8777d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc082dd6e00, {{0xc03e8777d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc03f622400, {0x63c85b8, 0xc05a1617d0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc05a1617d0}, {0x64a6560, 0xc082dd6e00}, {0xc03e8777d0, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc05a1617d0}, {0x6466ed8, 0xc05a2d9b80}, {0xc03e8777d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc074391fb0}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc05be31100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020af3400, {0x63c85b8, 0xc05a1617d0}, 0xc05be31100, 0xc077a2c640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020af3400, {0x63c85b8, 0xc05a1617d0}, 0xc05be31100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05a161740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05a161740}, {0xc07d2568f0, 0x660089}, 0xc07d2568f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05a161740}, {0xc00102e4e0}, 0xc05be31000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05a1616e0}, 0xc05be31000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06b4b8b10}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803247 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc0778ddec0}, 0xc01728ccb0, {0x646cc48, 0xc0778ddf80}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc0778ddec0}, 0xedb7a834f, 0xc04b7a8480) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc0778ddec0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc044d7cff0}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc01728cef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc044d7cff0}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219809663 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc073da5140}, 0xc03ed4a018, 0x1, 0x0, 0x1, 0xc046636de0, 0xc086f69da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc073da5140}, 0x120, 0xc033f3e728, 0x46636d80, 0x0, 0x660069, 0xc046636de0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc073da5140}, 0xc046636d80, {{{{0x0, 0x0}, {0xc05212a240, 0x4b}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc073da5140}, 0x6352410, 0x6b245110) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc073da5140}, {0xc046636900, {0x1743011065ed6dda, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc073da5140}, 0xc05ac772c0, 0xc0d1bd4ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc073da5140}, 0x660089, {0xc046636900, {0x1743011065ed6dda, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc073da5140}, 0xc05a12b600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011065ed6dda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011065ed6dda, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc073da50b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc073da50b0}, {0xc09365e8f0, 0x660089}, 0xc09365e8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc073da50b0}, {0xc00102e4e0}, 0xc05a12b500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc073da5050}, 0xc05a12b500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc073da4c90}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219811333 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc070269590}, 0xc056b273f8, 0x1, 0x70801811d17c7f, 0x37e11d600, 0xc04cae7140, 0xc06cb4c600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc070269590}, 0x1e0, 0xc01e91d230, 0x4cae70e0, 0x0, 0x660069, 0xc04cae7140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc070269590}, 0xc04cae70e0, {{{{0x0, 0x0}, {0xc095aac900, 0x112}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc070269590}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc070269590}, {0xc03acb8d80, {0x1743011074c5c0ad, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc070269590}, 0xc0129afea0, 0xc01e91d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc070269590}, 0x660089, {0xc03acb8d80, {0x1743011074c5c0ad, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc070269590}, 0xc0659cc900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011074c5c0ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011074c5c0ad, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070269500}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070269500}, {0xc01e91f3f8, 0x660089}, 0xc01e91f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070269500}, {0x7fe3f1f85418}, 0xc0659cc700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070269470}, 0xc0659cc700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}, 0xc047f1d5e0, 0xc056b273c8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}, 0x3, 0xc06348fac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc01e91f980}}}, {0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}, 0xc047f1d5e0, 0xc06348fac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070269470}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070269470}, {0x40, 0xc01e91fb70}, 0xc01e91fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}, 0x4beac80, 0x7fe457d92a01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070269470}, {0x4e9d7a0, 0xc0659cc700}, 0xc047f1d5e0, 0xc056b273c8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070269470}, 0xc028e9c4e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03acb8c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc03acb8c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812753 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219428202 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc078a458d0, 0x228) GOROOT/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc035fd6000) GOROOT/src/sync/cond.go:56 +0x8c github.com/cockroachdb/cockroach/pkg/sql.(*StmtBuf).CurCmd(0xc035fd6660) github.com/cockroachdb/cockroach/pkg/sql/conn_io.go:453 +0xbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc057e52600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1842 +0xbb github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc057e52600, {0x63c8510, 0xc078a45900}, 0xc09297ddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08fbe25a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc078a45900}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc08fbe25a0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811894 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02c4a03c0, {0x63c85b8, 0xc03298ca50}, 0xc021241f68, 0x1, 0x7080181915ae47, 0x37e11d600, 0xc06e7d1140, 0xc025b81da0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc03298ca50}, 0x120, 0xc033d53230, 0x6e7d10e0, 0x0, 0x660069, 0xc06e7d1140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02c4a03c0, {0x63c85b8, 0xc03298ca50}, 0xc06e7d10e0, {{{{0x0, 0x0}, {0xc035cd7200, 0x33}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03298ca50}, 0x6352410, 0x6b066668) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02c458d70, {0x63c85b8, 0xc03298ca50}, {0xc06e7d0ea0, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02c458d70, {0x63c85b8, 0xc03298ca50}, 0xc06aebb4a0, 0xc033d539f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03298ca50}, 0x660089, {0xc06e7d0ea0, {0x1743011062287dc1, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02c4a8c00, {0x63c85b8, 0xc03298ca50}, 0xc04c24af00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011062287dc1, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03298c990}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03298c990}, {0xc033d553f8, 0x660089}, 0xc033d55400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03298c990}, {0x2}, 0xc04c24ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03298c900}, 0xc04c24ad00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}, 0xc08d07b420, 0xc021241f38) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}, 0x3, 0xc0a43dc780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc033d55980}}}, {0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}, 0xc08d07b420, 0xc0a43dc780) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03298c900}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03298c900}, {0x40, 0xc033d55b70}, 0xc033d55ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03298c900}, {0x4e9d7a0, 0xc04c24ad00}, 0xc08d07b420, 0xc021241f38) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03298c900}, 0xc0cc003380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06e7d0d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc06e7d0d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219783356 [IO wait]: internal/poll.runtime_pollWait(0x7fdbaff36750, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc066237600, 0xc061ad9000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc066237600, {0xc061ad9000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc066237600, {0xc061ad9000, 0x735, 0xc068193400}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc062d857e0, {0xc061ad9000, 0x7eda79, 0xc061ad9005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04795cba0, {0xc061ad9000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04fe47da0, {0xc061ad9000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06414fe78, {0x631b900, 0xc04fe47da0}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06414fc00, {0x7fe42ab59310, 0xc04795cba0}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06414fc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06414fc00, {0xc086243000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc028f96600, {0xc086243000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc002a11588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc002a11588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc002a11b60, {0x634aac0, 0xc002a11588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc002a11500, 0xc0459e77d0, 0xc07395342b, 0xc07395342d, {0x635bec0, 0xc06f48c5a0}, 0xc000d6ae00, 0xc05ffe9cf4, 0xc073953429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc002a11500, {0x63c85b8, 0xc0756df4d0}, 0xc0459e77c0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc084b56038, 0x8}}, 0x0, 0xc0756df470, 0xc0756df4a0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0756df4d0}, {0x643d810, 0xc04795cba0}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc072103d10}, {0x643d810, 0xc04795cba0}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc072103d10}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219798575 [runnable]: sync.runtime_SemacquireMutex(0xc086138690, 0x8f, 0x7fddaa652ac0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc086138538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc086138560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0be98b200, {0xffffffffffffff15, 0xc0861386c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0be98b200) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0be98b200, {0xc0e7384600, 0x32, 0x71}, {0xc048cb4e40, 0x32, 0x5d}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc073b43298, {0xc0e7384600, 0x32, 0x71}, {0xc048cb4e40, 0x32, 0x5d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc073b41bc0, {0xc048cb4e40, 0x5d, 0x71}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc073b41bc0, {0xc0e7384600, 0x5d, 0xc048cb4e40}, {0xc048cb4e40, 0xc08a9c6820, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc073b41900, {0xc048cb4e40, 0x32, 0x5d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc05f376ea8, {{0xc08a9c6820, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc061330800, {{0xc0278f88d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc062810400, {0x63c85b8, 0xc06fe7f140}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06fe7f140}, {0x64a6560, 0xc061330800}, {0xc0278f88d0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06fe7f140}, {0x6466ed8, 0xc05f376c00}, {0xc0278f88d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc021ff7218}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc007711d00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020544800, {0x63c85b8, 0xc06fe7f140}, 0xc007711d00, 0xc0861d1a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020544800, {0x63c85b8, 0xc06fe7f140}, 0xc007711d00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06fe7f0b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06fe7f0b0}, {0xc002fea8f0, 0x660089}, 0xc002fea8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06fe7f0b0}, {0xc00102e4e0}, 0xc007711c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06fe7f050}, 0xc007711c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07aa4eff0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801881 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc039dac870}, 0xc0672e8e40, 0x1, 0xc0526b70a8, 0x519bbc, 0xc084c74180, 0xc00d80b6e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc039dac870}, 0x120, 0xc0526b7230, 0x84c74120, 0x0, 0x660069, 0xc084c74180, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc039dac870}, 0xc084c74120, {{{{0x0, 0x0}, {0xc03d2efd40, 0xee}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc039dac870}, 0x6352410, 0x1a06ebf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc039dac870}, {0xc084c74000, {0x174301101bfb2ecd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc039dac870}, 0xc04adf61e0, 0xc0526b79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc039dac870}, 0x660089, {0xc084c74000, {0x174301101bfb2ecd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc039dac870}, 0xc07c619a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101bfb2ecd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101bfb2ecd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc039dac780}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039dac780}, {0xc0526b93f8, 0x660089}, 0xc0526b9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc039dac780}, {0x7fe423ec24a0}, 0xc07c619700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc039dac6f0}, 0xc07c619700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}, 0xc019f5b480, 0xc018b01cb0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}, 0x3, 0xc05fa35840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0526b9980}}}, {0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}, 0xc019f5b480, 0xc05fa35840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc039dac6f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc039dac6f0}, {0x40, 0xc0526b9b70}, 0xc0526b9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc039dac6f0}, {0x4e9d7a0, 0xc07c619700}, 0xc019f5b480, 0xc018b01cb0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc039dac6f0}, 0xc0652100c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0424e5e60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0424e5e60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812437 [runnable]: sync.runtime_SemacquireMutex(0xa4a44a, 0xc0, 0x7fe3667fd660) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x459e98, 0x7fe3e7c9f000, 0xc02a0dae78) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x118663c, 0xc02a0daed0, 0xc02a0daeb0, 0x4bb045) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x7248b00, 0x2da4}, 0x0, 0xc060cd2278) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc060cd2000, {0xffffffffffffff15, 0xc02a0db040}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc060cd2000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc060cd2000, {0xc06f5cf500, 0x2e, 0x52}, 0x13, 0x13) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekGE(0xc060cd2000, {0xc06f5cf500, 0x2e, 0x52}, 0xe0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:513 +0xaf github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekGE(0xc060cd2000, {0xc06f5cf500, 0x2e, 0x52}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1317 +0x29d github.com/cockroachdb/pebble.(*levelIter).SeekGE(0xc093324d98, {0xc06f5cf500, 0x2e, 0x52}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:409 +0xd6 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0933236c0, {0xc06f5cf500, 0x60, 0x52}, 0x2e, 0x2e) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:844 +0x183 github.com/cockroachdb/pebble.(*mergingIter).SeekGE(0xc0933236c0, {0xc06f5cf500, 0x2e, 0xc02a82af00}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:899 +0x8c github.com/cockroachdb/pebble.(*Iterator).SeekGEWithLimit(0xc093323400, {0xc06f5cf500, 0x2e, 0x52}, {0xc0393c4cc0, 0x30, 0x60}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:929 +0x4bc github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0963edf78, {{0xc0a300aff0, 0x2d, 0xf0}, {0x0, 0x0, 0x0}}, {0xc0393c4cc0, 0x2f, 0x60}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:334 +0x173 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc00d8ee200, {{0xc03ea07dd0, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).iterSeek(0xc04979f800, {{0xc03ea07dd0, 0x26, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:1065 +0x71 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).scan(0xc04979f800, {0x63c85b8, 0xc067eb1d70}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:431 +0x186 github.com/cockroachdb/cockroach/pkg/storage.mvccScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc03ea07e00, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2283 +0x6ae github.com/cockroachdb/cockroach/pkg/storage.MVCCScanToBytes({_, _}, {_, _}, {_, _, _}, {0xc03ea07e00, 0x26, 0x30}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:2504 +0x2c6 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Scan({_, _}, {_, _}, {{0x64d25c8, 0xc079319bf0}, {{0x1743011082b57de8, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_scan.go:62 +0x41b github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011082b57de8, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc01c385500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110a082e2e8, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e06bc00, {0x63c85b8, 0xc067eb1d70}, 0xc01c385500, 0xc04ec39680) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e06bc00, {0x63c85b8, 0xc067eb1d70}, 0xc01c385500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011082b57de8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011082b57de8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc067eb1ce0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067eb1ce0}, {0xc02a0e13f8, 0x660089}, 0xc02a0e1400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc067eb1ce0}, {0xc03b626530}, 0xc01c385300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc067eb1c50}, 0xc01c385300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}, 0xc074682680, 0xc018c96660) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}, 0x3, 0xc0746f4100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc02a0e1980}}}, {0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}, 0xc074682680, 0xc0746f4100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc067eb1c50}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc067eb1c50}, {0x40, 0xc02a0e1b70}, 0xc02a0e1ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc067eb1c50}, {0x4e9d7a0, 0xc01c385300}, 0xc074682680, 0xc018c96660) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc067eb1c50}, 0xc020fe5080, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05b8dc7e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05b8dc7e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808181 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc066820fc0}, 0xc03ac61f20, 0x1, 0xc0070eb0a8, 0x519bbc, 0xc089a55e60, 0xc065130240) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc066820fc0}, 0x1e0, 0xc0070eb230, 0x89a55e00, 0x0, 0x660069, 0xc089a55e60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc066820fc0}, 0xc089a55e00, {{{{0x0, 0x0}, {0xc041e0d8c0, 0x39}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0xc098d71e60, {0x63c85b8, 0xc066820fc0}, 0xc06f36e4b0, 0x70eb428) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc066820fc0}, {0xc075bc3b00, {0x174301105aa6620c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc066820fc0}, 0xc0720b10e0, 0xc0070eb9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc066820fc0}, 0x660089, {0xc075bc3b00, {0x174301105aa6620c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc066820fc0}, 0xc09c378f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aa6620c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aa6620c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc066820f30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066820f30}, {0xc0070ed3f8, 0x660089}, 0xc0070ed400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc066820f30}, {0x7fe350a41758}, 0xc09c378d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc066820ea0}, 0xc09c378d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}, 0xc0156d06a0, 0xc04309f368) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}, 0x3, 0xc066f94ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0070ed980}}}, {0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}, 0xc0156d06a0, 0xc066f94ac0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc066820ea0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc066820ea0}, {0x40, 0xc0070edb70}, 0xc0070edab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc066820ea0}, {0x4e9d7a0, 0xc09c378d00}, 0xc0156d06a0, 0xc04309f368) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc066820ea0}, 0xc0309611a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075bc39e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc075bc39e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219808128 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0210c1c00, {0x63c85b8, 0xc03f2981b0}, 0xc084143700, 0xc07ad9ca00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0210c1c00, {0x63c85b8, 0xc03f2981b0}, 0xc084143700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105c49901c, 0x0, 0x0}, 0xc092d5fdf0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105c49901c, 0x0, 0x0}, 0xc092d5fdf0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03f298120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f298120}, {0xc00b0f93f8, 0x660089}, 0xc00b0f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03f298120}, {0x556e67312b795a53}, 0xc084143500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03f298090}, 0xc084143500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}, 0xc072317300, 0xc0654a6df8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}, 0x3, 0xc057a64540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b0f9980}}}, {0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}, 0xc072317300, 0xc057a64540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03f298090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f298090}, {0x40, 0xc00b0f9b70}, 0xc00b0f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03f298090}, {0x4e9d7a0, 0xc084143500}, 0xc072317300, 0xc0654a6df8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03f298090}, 0xc072d08de0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059e98c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc059e98c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219798863 [runnable]: sync.runtime_SemacquireMutex(0xc07d1a0690, 0x8f, 0xc07d1a06a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc07d1a0538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc07d1a0560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0085ee000, {0xffffffffffffff15, 0xc07d1a06c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0085ee000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0085ee000, {0xc01c8ccb40, 0x32, 0x32}, {0xc00ac68b00, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091419998, {0xc01c8ccb40, 0x32, 0x32}, {0xc00ac68b00, 0x32, 0x79}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0914182c0, {0xc00ac68b00, 0x79, 0x32}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0914182c0, {0xc01c8ccb40, 0x79, 0xc00ac68b00}, {0xc00ac68b00, 0xc02c8e0300, 0x31}, 0x40) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091418000, {0xc00ac68b00, 0x32, 0x79}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0520382a8, {{0xc02c8e0300, 0x31, 0x40}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc09c55ec00, {{0xc02b8c4510, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc053cfb400, {0x63c85b8, 0xc042f17590}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc042f17590}, {0x64a6560, 0xc09c55ec00}, {0xc02b8c4510, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc042f17590}, {0x6466ed8, 0xc052038000}, {0xc02b8c4510, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc066835140}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc061baa800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020798c00, {0x63c85b8, 0xc042f17590}, 0xc061baa800, 0xc07c338fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020798c00, {0x63c85b8, 0xc042f17590}, 0xc061baa800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc042f174a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc042f174a0}, {0xc0492c88f0, 0x660089}, 0xc0492c88f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc042f174a0}, {0xc00102e4e0}, 0xc061baa700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc042f17440}, 0xc061baa700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc038bba6f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219799852 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc04ba7fa40}, 0xc033e93b00, 0xc01267e460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04ba7fa40}, 0xc033e93b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100c6d2837, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100c6d2837, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ba7f980}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ba7f980}, {0xc04e1ab3f8, 0x660089}, 0xc04e1ab400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ba7f980}, {0x64ce030}, 0xc033e93700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ba7f8f0}, 0xc033e93700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}, 0xc07a8d1400, 0xc03de715d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}, 0x3, 0xc018811980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04e1ab980}}}, {0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}, 0xc07a8d1400, 0xc018811980) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ba7f8f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ba7f8f0}, {0x40, 0xc04e1abb70}, 0xc04e1abab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ba7f8f0}, {0x4e9d7a0, 0xc033e93700}, 0xc07a8d1400, 0xc03de715d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ba7f8f0}, 0xc0ccee9f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0662af7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc0662af7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812601 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219721221 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafea2e60, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc067a09500, 0xc085592800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc067a09500, {0xc085592800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc067a09500, {0xc085592800, 0x739, 0xc0367837c0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01e0a3c28, {0xc085592800, 0x100, 0xc085592805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04b689260, {0xc085592800, 0x1, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc05bebf680, {0xc085592800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc06ababe78, {0x631b900, 0xc05bebf680}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc06ababc00, {0x7fe42ab59310, 0xc04b689260}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc06ababc00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc06ababc00, {0xc087038000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc0148c8e88, {0xc087038000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0404fcb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0404fcb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0404fd0e0, {0x634aac0, 0xc0404fcb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0404fca80, 0xc0456263f0, 0xc09670542b, 0xc09670542d, {0x635bec0, 0xc01d6e81b0}, 0xc000d6ae00, 0xc05730f078, 0xc096705429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0404fca80, {0x63c85b8, 0xc07037a150}, 0xc0456263e0, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc00c084043, 0xb}}, 0x0, 0xc07037a0c0, 0xc07037a0f0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc07037a150}, {0x643d810, 0xc04b689260}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0516bfc20}, {0x643d810, 0xc04b689260}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0516bfc20}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812641 [runnable]: internal/poll.runtime_pollWait(0x7fe38e3e0040, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06d853d80, 0xc0a57e6900, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06d853d80, {0xc0a57e6900, 0x205, 0x205}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06d853d80, {0xc0a57e6900, 0x0, 0x0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc033a92da8, {0xc0a57e6900, 0x0, 0x205}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04d73b140, {0xc0a57e6900, 0x4da9180, 0x205}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc04a6d9c80, {0xc0a57e6900, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0787b3078, {0x631b900, 0xc04a6d9c80}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0787b2e00, {0x7fe42ab59310, 0xc04d73b140}, 0xc3) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0787b2e00, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).readHandshake(0xc0787b2e00) GOROOT/src/crypto/tls/conn.go:1010 +0x6d crypto/tls.(*Conn).readClientHello(0xc0787b2e00, {0x63c8510, 0xc050a07480}) GOROOT/src/crypto/tls/handshake_server.go:132 +0x45 crypto/tls.(*Conn).serverHandshake(0xc0787b2e00, {0x63c8510, 0xc050a07480}) GOROOT/src/crypto/tls/handshake_server.go:43 +0x46 crypto/tls.(*Conn).handshakeContext(0xc0787b2e00, {0x63c8548, 0xc000140068}) GOROOT/src/crypto/tls/conn.go:1453 +0x36f crypto/tls.(*Conn).HandshakeContext(...) GOROOT/src/crypto/tls/conn.go:1396 crypto/tls.(*Conn).Handshake(...) GOROOT/src/crypto/tls/conn.go:1380 crypto/tls.(*Conn).Read(0xc0787b2e00, {0xc073d14df8, 0x4, 0xc019549520}) GOROOT/src/crypto/tls/conn.go:1265 +0x65 io.ReadAtLeast({0x7fe456a34450, 0xc0787b2e00}, {0xc073d14df8, 0x4, 0x4}, 0x4) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadUntypedMsg(0xc073d14de0, {0x7fe456a34450, 0xc0787b2e00}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:151 +0x65 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).readVersion(0xc00c689e00, {0x7fe456a34450, 0xc0787b2e00}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1357 +0x11f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).maybeUpgradeToSecureConn(0xc00c689e00, {0x0, 0xffc}, {0x643d810, 0xc04d73b140}, 0x2, 0x4d2162f, 0xc003107be0) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1301 +0x367 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c8510, 0xc050a07440}, {0x643d810, 0xc04d73b140}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:807 +0x5a9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc073d14d20}, {0x643d810, 0xc04d73b140}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc073d14d20}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806168 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc04c5e3140}, 0xc08aa69b48, 0x1, 0x708017e8d45300, 0x37e11d600, 0xc057118ba0, 0xc0571189c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc04c5e3140}, 0x1e0, 0xc007545230, 0x57118b40, 0x0, 0x660069, 0xc057118ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc04c5e3140}, 0xc057118b40, {{{{0x0, 0x0}, {0xc0765b30e0, 0x16}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc04c5e3140}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc04c5e3140}, {0xc084c6d680, {0x174301104be59ad3, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc04c5e3140}, 0xc0436ce820, 0xc0075459f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04c5e3140}, 0x660089, {0xc084c6d680, {0x174301104be59ad3, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc04c5e3140}, 0xc09cdb5900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104be59ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104be59ad3, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c5e3050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c5e3050}, {0xc0075473f8, 0x660089}, 0xc007547400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c5e3050}, {0x3}, 0xc09cdb5600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c5e2fc0}, 0xc09cdb5600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}, 0xc0762ae7a0, 0xc08aa69b18) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}, 0x3, 0xc070fe33c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc007547980}}}, {0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}, 0xc0762ae7a0, 0xc070fe33c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04c5e2fc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c5e2fc0}, {0x40, 0xc007547b70}, 0xc007547ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04c5e2fc0}, {0x4e9d7a0, 0xc09cdb5600}, 0xc0762ae7a0, 0xc08aa69b18) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04c5e2fc0}, 0xc041671380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc084c6d560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc084c6d560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812700 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219806897 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc09293cc00}, 0xc020748570, 0x1, 0xc004e690a8, 0x519bbc, 0xc01bacf140, 0xc03631c180) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc09293cc00}, 0x120, 0xc004e69230, 0x1bacf0e0, 0x0, 0x660069, 0xc01bacf140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc09293cc00}, 0xc01bacf0e0, {{{{0x0, 0x0}, {0xc09c674d80, 0x28}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc09293cc00}, 0x6352410, 0x304bbc88) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc09293cc00}, {0xc01bacefc0, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc09293cc00}, 0xc04def8c80, 0xc004e699f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc09293cc00}, 0x660089, {0xc01bacefc0, {0x1743011050527b3e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc09293cc00}, 0xc076736900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011050527b3e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc09293cb70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09293cb70}, {0xc004e6b3f8, 0x660089}, 0xc004e6b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc09293cb70}, {0x7fe426b1ded8}, 0xc076736700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc09293cae0}, 0xc076736700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}, 0xc08e0fa880, 0xc020748a68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}, 0x3, 0xc08fe0a740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004e6b980}}}, {0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}, 0xc08e0fa880, 0xc08fe0a740) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc09293cae0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc09293cae0}, {0x40, 0xc004e6bb70}, 0xc004e6bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc09293cae0}, {0x4e9d7a0, 0xc076736700}, 0xc08e0fa880, 0xc020748a68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc09293cae0}, 0xc03c46ce40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc000b4b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc000b4b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803454 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc025584840}, 0xc014a117a0, 0x1, 0x0, 0x1, 0xc03c75f140, 0xc04ace22a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc025584840}, 0x120, 0xc0528a2728, 0x3c75f0e0, 0x0, 0x660069, 0xc03c75f140, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc025584840}, 0xc03c75f0e0, {{{{0x0, 0x0}, {0xc086d967e0, 0x1d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc025584840}, 0x6352410, 0x343304b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc025584840}, {0xc03b940c60, {0x174301100d71b895, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc025584840}, 0xc05ad9e5a0, 0xc07c43eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc025584840}, 0x660089, {0xc03b940c60, {0x174301100d71b895, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc025584840}, 0xc052c6e300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100d71b895, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100d71b895, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc025584780}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc025584780}, {0xc0936968f0, 0x660089}, 0xc0936968f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc025584780}, {0xc00102e4e0}, 0xc052c6e200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc025584720}, 0xc052c6e200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc037feab70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805239 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0887ce210}, 0xc0269d7890, 0x1, 0xc07ae4b0a8, 0x519bbc, 0xc059e16240, 0xc0595dcf60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0887ce210}, 0x1e0, 0xc07ae4b230, 0x59e161e0, 0xc0bb2b04e0, 0x18, 0xc059e16240, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0887ce210}, 0xc059e161e0, {{{{0x0, 0x0}, {0xc0595e38c0, 0x53}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0887ce210}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0887ce210}, {0xc059e64120, {0x174301103f85c3dd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0887ce210}, 0xc068090140, 0xc07ae4b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0887ce210}, 0x660089, {0xc059e64120, {0x174301103f85c3dd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0887ce210}, 0xc049c64500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103f85c3dd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103f85c3dd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0887ce180}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0887ce180}, {0xc07ae4d3f8, 0x660089}, 0xc07ae4d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0887ce180}, {0x0}, 0xc049c64300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0887ce0f0}, 0xc049c64300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}, 0xc0654e2da0, 0xc0193160c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}, 0x3, 0xc08706d840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07ae4d980}}}, {0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}, 0xc0654e2da0, 0xc08706d840) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0887ce0f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0887ce0f0}, {0x40, 0xc07ae4db70}, 0xc07ae4dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}, 0x203021, 0x203021) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0887ce0f0}, {0x4e9d7a0, 0xc049c64300}, 0xc0654e2da0, 0xc0193160c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0887ce0f0}, 0xc059cf0420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc059e64000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc059e64000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809907 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc011da9800, {0x63c85b8, 0xc01e8320c0}, 0xc084513400, 0xc07330fea0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc011da9800, {0x63c85b8, 0xc01e8320c0}, 0xc084513400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011067f96ebd, 0x0, 0x0}, 0xc031cfcc40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011067f96ebd, 0x0, 0x0}, 0xc031cfcc40, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01e832030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01e832030}, {0xc00df473f8, 0x660089}, 0xc00df47400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01e832030}, {0x0}, 0xc084513100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046147f80}, 0xc084513100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}, 0xc056c911a0, 0xc01b3ea930) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}, 0x3, 0xc03f5d3340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00df47980}}}, {0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}, 0xc056c911a0, 0xc03f5d3340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046147f80}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046147f80}, {0x40, 0xc00df47b70}, 0xc00df47ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046147f80}, {0x4e9d7a0, 0xc084513100}, 0xc056c911a0, 0xc01b3ea930) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046147f80}, 0xc01ab37f80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07c7177a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07c7177a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810285 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02bfb0c00, {0x63c85b8, 0xc03bf1d2f0}, 0xc08ad6f600, 0xc0467a1360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02bfb0c00, {0x63c85b8, 0xc03bf1d2f0}, 0xc08ad6f600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106cf7306b, 0x0, 0x0}, 0xc03905a7a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106cf7306b, 0x0, 0x0}, 0xc03905a7a0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03bf1d260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bf1d260}, {0xc0294773f8, 0x660089}, 0xc029477400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03bf1d260}, {0xc0015221e0}, 0xc08ad6f400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03bf1d1a0}, 0xc08ad6f400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}, 0xc07252ac80, 0xc03d272ea0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}, 0x3, 0xc073feb540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc029477980}}}, {0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}, 0xc07252ac80, 0xc073feb540) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03bf1d1a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03bf1d1a0}, {0x40, 0xc029477b70}, 0xc029477ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03bf1d1a0}, {0x4e9d7a0, 0xc08ad6f400}, 0xc07252ac80, 0xc03d272ea0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03bf1d1a0}, 0xc0b59fe1e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060f5b200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc060f5b200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810398 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc021def540, {0x63c85b8, 0xc0310159b0}, 0xc064121548, 0x1, 0x0, 0x1, 0xc05338d6e0, 0xc0786ee4e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0310159b0}, 0x120, 0xc01fc8c728, 0x5338d680, 0x0, 0x660069, 0xc05338d6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc021def540, {0x63c85b8, 0xc0310159b0}, 0xc05338d680, {{{{0x0, 0x0}, {0xc05ebc45a0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0310159b0}, 0x6352410, 0x50b34d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc021d3dcc0, {0x63c85b8, 0xc0310159b0}, {0xc0188eaa20, {0x174301106d30e63a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc021d3dcc0, {0x63c85b8, 0xc0310159b0}, 0xc07c1fb0e0, 0xc00b2c0ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0310159b0}, 0x660089, {0xc0188eaa20, {0x174301106d30e63a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc021ebe400, {0x63c85b8, 0xc0310159b0}, 0xc08a0b2400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d30e63a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d30e63a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc031015740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc031015740}, {0xc0936388f0, 0x660089}, 0xc0936388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc031015740}, {0xc00102e4e0}, 0xc08a0b2300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0310156e0}, 0xc08a0b2300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc012bc1290}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805046 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02a151680, {0x63c85b8, 0xc0701d1920}, 0xc01fb0aeb8, 0x1, 0x708017da5cd325, 0x37e11d600, 0xc0575c8060, 0xc03b2cd500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0701d1920}, 0x51cdcd8, 0xc07c353230, 0x7c353138, 0x4c166a, 0x18, 0xc0575c8060, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02a151680, {0x63c85b8, 0xc0701d1920}, 0xc0575c8000, {{{{0x0, 0x0}, {0xc055d3f8c0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0701d1920}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02a12cf50, {0x63c85b8, 0xc0701d1920}, {0xc070651c20, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02a12cf50, {0x63c85b8, 0xc0701d1920}, 0xc070ba23c0, 0xc07c3539f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0701d1920}, 0x660089, {0xc070651c20, {0x17430110245855e8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02a194000, {0x63c85b8, 0xc0701d1920}, 0xc063fc1600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110245855e8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0701d1890}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0701d1890}, {0xc07c3553f8, 0x660089}, 0xc07c355400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0701d1890}, {0xc0015221e0}, 0xc063fc1300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0701d16e0}, 0xc063fc1300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}, 0xc065a4b220, 0xc01fb0ae58) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}, 0x3, 0xc00e1e8c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07c355980}}}, {0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}, 0xc065a4b220, 0xc00e1e8c80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0701d16e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0701d16e0}, {0x40, 0xc07c355b70}, 0xc07c355ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}, 0x203003, 0x203003) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0701d16e0}, {0x4e9d7a0, 0xc063fc1300}, 0xc065a4b220, 0xc01fb0ae58) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0701d16e0}, 0xc01838e8a0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b560, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc07013b560, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802487 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc0169d1e60}, 0xc07168b800, 0xc02667e000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0169d1e60}, 0xc07168b800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011022fb1f8a, 0x0, 0x0}, 0xc03fa9f170, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011022fb1f8a, 0x0, 0x0}, 0xc03fa9f170, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0169d1bf0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0169d1bf0}, {0xc05b0e53f8, 0x660089}, 0xc05b0e5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0169d1bf0}, {0x0}, 0xc07168b600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0169d1aa0}, 0xc07168b600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}, 0xc02aa687e0, 0xc056347a70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}, 0x3, 0xc08451eec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc05b0e5980}}}, {0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}, 0xc02aa687e0, 0xc08451eec0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0169d1aa0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0169d1aa0}, {0x40, 0xc05b0e5b70}, 0xc05b0e5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0169d1aa0}, {0x4e9d7a0, 0xc07168b600}, 0xc02aa687e0, 0xc056347a70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0169d1aa0}, 0xc023901800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc012cecb40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc012cecb40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810029 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc04fc4a000}, 0xc0286ff158, 0x1, 0xc00771d0a8, 0x519bbc, 0xc0080fbac0, 0xc03242f080) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0614b0c60, {0x63c85b8, 0xc04fc4a000}, 0xc0614b0cf8, 0xc00771d1a8, 0x27, 0x40, 0xffffffffffffffff, 0xc0080fbac0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc04fc4a000}, 0xc0080fb880, {{{{0xc05fad0c60, 0x10}, {0xc032b01680, 0x24}}, {{0xc0283066c0, 0x42}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04fc4a000}, 0x63521a0, 0x412ac500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc04fc4a000}, {0xc086bc6ea0, {0x1743010ff47ee81f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc04fc4a000}, 0xc03ecb7540, 0xc00771d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04fc4a000}, 0x660089, {0xc086bc6ea0, {0x1743010ff47ee81f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc04fc4a000}, 0xc08ad40700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ff47ee81f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ff47ee81f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc076483e30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076483e30}, {0xc00771f3f8, 0x660089}, 0xc00771f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc076483e30}, {0xc00771f478}, 0xc08ad40500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc076483d10}, 0xc08ad40500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}, 0xc018a6c2c0, 0xc02463a078) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}, 0x3, 0xc041243c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00771f980}}}, {0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}, 0xc018a6c2c0, 0xc041243c40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc076483d10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc076483d10}, {0x40, 0xc00771fb70}, 0xc00771fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc076483d10}, {0x4e9d7a0, 0xc08ad40500}, 0xc018a6c2c0, 0xc02463a078) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc076483d10}, 0xc082184ba0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01e819200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01e819200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812731 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03bd19180, {0x63c85b8, 0xc062cdf230}, 0xc0490963a8, 0x1, 0x7080183b6cc874, 0x37e11d600, 0xc0588e91a0, 0xc06425ad20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc093edd0e0, {0x63c85b8, 0xc062cdf230}, 0xc0588e91a0, 0xc03c6591a8, 0x45, 0x60, 0x9, 0xc0588e91a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03bd19180, {0x63c85b8, 0xc062cdf230}, 0xc0588e90e0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc079078900, 0xe}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc062cdf230}, 0x6352260, 0x3abf6b28) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc067985680, {0x63c85b8, 0xc062cdf230}, {0xc0588e8fc0, {0x174301105b77d610, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc067985680, {0x63c85b8, 0xc062cdf230}, 0xc083df4960, 0xc03c6599f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc062cdf230}, 0x660089, {0xc0588e8fc0, {0x174301105b77d610, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc061dce000, {0x63c85b8, 0xc062cdf230}, 0xc050ef3b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b77d610, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cdf1a0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdf1a0}, {0xc03c65b3f8, 0x660089}, 0xc03c65b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cdf1a0}, {0x2002000010090000}, 0xc050ef3900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cdf110}, 0xc050ef3900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}, 0xc06ac12d00, 0xc049096378) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}, 0x3, 0xc0b08ec240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc03c65b980}}}, {0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}, 0xc06ac12d00, 0xc0b08ec240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cdf110}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cdf110}, {0x40, 0xc03c65bb70}, 0xc03c65bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cdf110}, {0x4e9d7a0, 0xc050ef3900}, 0xc06ac12d00, 0xc049096378) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cdf110}, 0xc049257e60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f8d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0218f8d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812866 [runnable]: internal/poll.runtime_pollWait(0x7fe422a59d60, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06481a600, 0xc0678f9440, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06481a600, {0xc0678f9440, 0x205, 0x205}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06481a600, {0xc0678f9440, 0x460ef4, 0x0}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc033a92e28, {0xc0678f9440, 0x0, 0x205}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc04d73b200, {0xc0678f9440, 0x0, 0x205}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0408cf0c8, {0xc0678f9440, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0bd7e5af8, {0x631b900, 0xc0408cf0c8}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0bd7e5880, {0x7fe42ab59310, 0xc04d73b200}, 0xc07a1be6c0) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0bd7e5880, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).readHandshake(0xc0bd7e5880) GOROOT/src/crypto/tls/conn.go:1010 +0x6d crypto/tls.(*Conn).readClientHello(0xc0bd7e5880, {0x63c8510, 0xc0a4ac25c0}) GOROOT/src/crypto/tls/handshake_server.go:132 +0x45 crypto/tls.(*Conn).serverHandshake(0xc0bd7e5880, {0x63c8510, 0xc0a4ac25c0}) GOROOT/src/crypto/tls/handshake_server.go:43 +0x46 crypto/tls.(*Conn).handshakeContext(0xc0bd7e5880, {0x63c8548, 0xc000140068}) GOROOT/src/crypto/tls/conn.go:1453 +0x36f crypto/tls.(*Conn).HandshakeContext(...) GOROOT/src/crypto/tls/conn.go:1396 crypto/tls.(*Conn).Handshake(...) GOROOT/src/crypto/tls/conn.go:1380 crypto/tls.(*Conn).Read(0xc0bd7e5880, {0xc044b77218, 0x4, 0xc07e11b520}) GOROOT/src/crypto/tls/conn.go:1265 +0x65 io.ReadAtLeast({0x7fe456a34450, 0xc0bd7e5880}, {0xc044b77218, 0x4, 0x4}, 0x4) GOROOT/src/io/io.go:328 +0x9a io.ReadFull(...) GOROOT/src/io/io.go:347 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadUntypedMsg(0xc044b77200, {0x7fe456a34450, 0xc0bd7e5880}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:151 +0x65 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).readVersion(0xc00c689e00, {0x7fe456a34450, 0xc0bd7e5880}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1357 +0x11f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).maybeUpgradeToSecureConn(0xc00c689e00, {0x0, 0xffc}, {0x643d810, 0xc04d73b200}, 0x2, 0x4d2162f, 0xc0110d7be0) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1301 +0x367 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c8510, 0xc0a4ac2580}, {0x643d810, 0xc04d73b200}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:807 +0x5a9 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc044b77140}, {0x643d810, 0xc04d73b200}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc044b77140}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812758 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803482 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc04793c4b0}, 0xc040840c30, 0x1, 0x0, 0x1, 0xc03b85e960, 0xc0388916e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04793c4b0}, 0x120, 0xc0762d8728, 0x3b85e900, 0x0, 0x660069, 0xc03b85e960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc04793c4b0}, 0xc03b85e900, {{{{0x0, 0x0}, {0xc0664bb8c0, 0x18}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04793c4b0}, 0x6352410, 0x14c75460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc04793c4b0}, {0xc03b85e120, {0x174301102ce103ca, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc04793c4b0}, 0xc0cc0b8f00, 0xc04c2f2ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04793c4b0}, 0x660089, {0xc03b85e120, {0x174301102ce103ca, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc04793c4b0}, 0xc00fe99e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102ce103ca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102ce103ca, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04793c420}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04793c420}, {0xc001c988f0, 0x660089}, 0xc001c988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04793c420}, {0xc00102e4e0}, 0xc00fe99d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04793c360}, 0xc00fe99d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc0162a6090}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219797444 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc066356d80}, 0xc007772cb0, {0x646cc48, 0xc066356de0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc066356d80}, 0xedb7a834e, 0xc0351bafc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc066356d80}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc055f0f500}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc007772ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc055f0f500}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804344 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc0863b4de0}, 0xc0159ba7f8, 0x1, 0xc079f750a8, 0x519bbc, 0xc05866db60, 0xc05928c4e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0863b4de0}, 0x120, 0xc079f75230, 0x5866db00, 0x0, 0x660069, 0xc05866db60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc0863b4de0}, 0xc05866db00, {{{{0x0, 0x0}, {0xc0437fcc60, 0x1a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0863b4de0}, 0x6352410, 0x1b25c0b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc0863b4de0}, {0xc05866d9e0, {0x174301103208748e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc0863b4de0}, 0xc06bd8eaa0, 0xc079f759f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0863b4de0}, 0x660089, {0xc05866d9e0, {0x174301103208748e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc0863b4de0}, 0xc040010600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103208748e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103208748e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0863b4d50}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0863b4d50}, {0xc079f773f8, 0x660089}, 0xc079f77400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0863b4d50}, {0xc03454fe90}, 0xc040010300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0863b4cc0}, 0xc040010300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}, 0xc04d7bba40, 0xc028353b00) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}, 0x3, 0xc06fa51680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079f77980}}}, {0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}, 0xc04d7bba40, 0xc06fa51680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0863b4cc0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0863b4cc0}, {0x40, 0xc079f77b70}, 0xc079f77ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0863b4cc0}, {0x4e9d7a0, 0xc040010300}, 0xc04d7bba40, 0xc028353b00) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0863b4cc0}, 0xc03db1f9e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05866d8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05866d8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219807374 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc05aaa6030}, 0xc051d74400, 0xc03e7d2fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc05aaa6030}, 0xc051d74400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b1ef448, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b1ef448, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0701c1f80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0701c1f80}, {0xc0172d27f8, 0x660089}, 0xc0172d2800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0701c1f80}, {0xc00102e4e0}, 0xc051d74300) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0701c1f20}, 0xc051d74300) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc0475ae4d0, {0x63c85b8, 0xc0701c1da0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc06e11d300, {0x63c85b8, 0xc0701c1da0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc06e11d300, {0x63c8510, 0xc059dfb000}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc06e11d300, {0x63c8510, 0xc059dfb000}, {0x63f7db8, 0x9bf6f48}, 0xc0172d7088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc06e11d300, {0x63c85b8, 0xc08a6031a0}, {{0x63f7db8, 0x9bf6f48}, {0xc0893c2cf8, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc06e11d300, {0x63c85b8, 0xc08a6031a0}, {{0x63f7db8, 0x9bf6f48}, {0xc0893c2cf8, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0893c2cf8, 0x6}, 0x0, 0x0}, {0x4880a65, 0xedb7a8350, 0x0}, {0x4881171, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc06e11d300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc06e11d300, {0x63c8510, 0xc08872bf80}, 0xc089d57dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc038af5330) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc08872bf80}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc038af5330) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219800614 [runnable]: sync.runtime_SemacquireMutex(0xc095fc8690, 0x8f, 0x7fe2c1526b80) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc095fc8538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc095fc8560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc072e34480, {0xffffffffffffff15, 0xc095fc86c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc072e34480) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc072e34480, {0xc022372e80, 0x32, 0x7f}, {0xc0160d0080, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07caf6d98, {0xc022372e80, 0x32, 0x7f}, {0xc0160d0080, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc07caf56c0, {0xc0160d0080, 0x74, 0x7f}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc07caf56c0, {0xc022372e80, 0x74, 0xc0160d0080}, {0xc0160d0080, 0xc08b522300, 0x31}, 0xc0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc07caf5400, {0xc0160d0080, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc051883428, {{0xc08b522300, 0x31, 0xc0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc083617f00, {{0xc082654ab0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc060925400, {0x63c85b8, 0xc06b9b20c0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06b9b20c0}, {0x64a6560, 0xc083617f00}, {0xc082654ab0, 0x2594d, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06b9b20c0}, {0x6466ed8, 0xc051883180}, {0xc082654ab0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0286fe858}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc057c0a300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020b0f800, {0x63c85b8, 0xc06b9b20c0}, 0xc057c0a300, 0xc03059d220) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020b0f800, {0x63c85b8, 0xc06b9b20c0}, 0xc057c0a300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06b9b2030}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06b9b2030}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06b9b2030}, {0xc00102e4e0}, 0xc057c0a200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc051401fb0}, 0xc057c0a200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc06b4b8db0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219804402 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e7208c0, {0x63c85b8, 0xc0210909f0}, 0xc0541fe330, 0x1, 0xc079e970a8, 0x519bbc, 0xc04de3dc80, 0xc04c4ad140) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0210909f0}, 0x1e0, 0xc079e97230, 0x4de3dc20, 0x0, 0x660069, 0xc04de3dc80, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e7208c0, {0x63c85b8, 0xc0210909f0}, 0xc04de3dc20, {{{{0x0, 0x0}, {0xc01e941440, 0x12}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0210909f0}, 0x63521a0, 0x321a0a18) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e62d5e0, {0x63c85b8, 0xc0210909f0}, {0xc05859f9e0, {0x174301103208748e, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e62d5e0, {0x63c85b8, 0xc0210909f0}, 0xc064d9efa0, 0xc079e979f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0210909f0}, 0x660089, {0xc05859f9e0, {0x174301103208748e, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e72bc00, {0x63c85b8, 0xc0210909f0}, 0xc039e60700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301103208748e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301103208748e, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021090960}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021090960}, {0xc079e993f8, 0x660089}, 0xc079e99400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021090960}, {0x4d939c}, 0xc039e60500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0210908d0}, 0xc039e60500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}, 0xc05393ec20, 0xc07a3e13f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}, 0x3, 0xc0cc382180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc079e99980}}}, {0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}, 0xc05393ec20, 0xc0cc382180) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0210908d0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0210908d0}, {0x40, 0xc079e99b70}, 0xc079e99ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0210908d0}, {0x4e9d7a0, 0xc039e60500}, 0xc05393ec20, 0xc07a3e13f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0210908d0}, 0xc04ad1c9c0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05859f8c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05859f8c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805285 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc03012e500, {0x63c85b8, 0xc0688d72f0}, 0xc032d840f0, 0x1, 0x708017dde4c75b, 0x37e11d600, 0xc05920f380, 0xc059ce0cc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0688d72f0}, 0x120, 0xc087992178, 0x5920f320, 0xc0109481a0, 0x18, 0xc05920f380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc03012e500, {0x63c85b8, 0xc0688d72f0}, 0xc05920f320, {{{{0x0, 0x0}, {0xc089989320, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0688d72f0}, 0x6352410, 0x17bb34e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02f553c20, {0x63c85b8, 0xc0688d72f0}, {0xc05920f200, {0x1743011040e63d92, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02f553c20, {0x63c85b8, 0xc0688d72f0}, 0xc051a2ec80, 0xc0966fe938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0688d72f0}, 0x660089, {0xc05920f200, {0x1743011040e63d92, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc030128800, {0x63c85b8, 0xc0688d72f0}, 0xc05897cb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011040e63d92, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011040e63d92, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0688d7260}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0688d7260}, {0xc096700340, 0x660089}, 0xc096700348) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0688d7260}, {0xc00102e4e0}, 0xc05897ca00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0688d7200}, 0xc05897ca00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc064734bd0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807306 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a6280, {0x63c85b8, 0xc043060f60}, 0xc05fe66e10, 0x1, 0xc0040270a8, 0x519bbc, 0xc082d0fe40, 0xc067bc9f20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc0664b0000, {0x63c85b8, 0xc043060f60}, 0xc082d0fe40, 0xc0040271a8, 0x41, 0x60, 0x0, 0xc082d0fe40, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a6280, {0x63c85b8, 0xc043060f60}, 0xc082d0fc00, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc073da6ea0, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc043060f60}, 0x63521a0, 0x5449bf38) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02dfddc70, {0x63c85b8, 0xc043060f60}, {0xc07b8f8240, {0x1743010ffc71a3f8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02dfddc70, {0x63c85b8, 0xc043060f60}, 0xc03a1f0c80, 0xc0040279f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc043060f60}, 0x660089, {0xc07b8f8240, {0x1743010ffc71a3f8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e09a800, {0x63c85b8, 0xc043060f60}, 0xc08d7f3400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffc71a3f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffc71a3f8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc043060ed0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043060ed0}, {0xc0040293f8, 0x660089}, 0xc004029400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc043060ed0}, {0x0}, 0xc08d7f3200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc043060d20}, 0xc08d7f3200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}, 0xc006dce660, 0xc0a55fe378) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}, 0x3, 0xc00ea59240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004029980}}}, {0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}, 0xc006dce660, 0xc00ea59240) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc043060d20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc043060d20}, {0x40, 0xc004029b70}, 0xc004029ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc043060d20}, {0x4e9d7a0, 0xc08d7f3200}, 0xc006dce660, 0xc0a55fe378) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc043060d20}, 0xc02269d800, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07b8f8120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc07b8f8120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810396 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc02822bf50}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219800135 [runnable]: sync.runtime_SemacquireMutex(0xc00074ab70, 0x44, 0xc09286e548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc09286e5b0, 0xc04f02a240, 0xc05581c000) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x1) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc04d163600, {0x116e025, 0xc089605df8}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0b510d180, {0xc02c62d080, 0x2b, 0x77}, {0xc06432e150, 0x2b, 0x6d}, 0x1, 0x5d) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc089605c90, {0xc02c62d080, 0x1, 0x6d}, {0xc06432e150, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc089605c90, {0xc02c62d080, 0x2b, 0x77}, {0xc06432e150, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc089604cc0, {0xc06432e150, 0x6d, 0x77}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc089604cc0, {0xc02c62d080, 0x2b, 0xc06432e150}, {0xc06432e150, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc089604a00, {0xc06432e150, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc091663088, {{0xc0160b6750, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc017f8f300, {{0xc0160b6750, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc07e2c6000, {0x63c85b8, 0xc02dd0a420}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc02dd0a420}, {0x64a6560, 0xc017f8f300}, {0xc0160b6750, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc02dd0a420}, {0x6466ed8, 0xc091663080}, {0xc0160b6750, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c77ff80}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046785100, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a30c00, {0x63c85b8, 0xc02dd0a420}, 0xc046785100, 0xc02ff160a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a30c00, {0x63c85b8, 0xc02dd0a420}, 0xc046785100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02dd0a300}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02dd0a300}, {0xc0036f68f0, 0x660089}, 0xc0036f68f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02dd0a300}, {0xc00102e4e0}, 0xc046785000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02dd0a120}, 0xc046785000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07464d2f0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812514 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0721f6cd0, {0x63c85b8, 0xc050b46c30}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc050b46c30}, {0x4f67ad2, 0x6446ff8}, 0xc07e11bfb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219804328 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01edaa500, {0x63c85b8, 0xc04fff5dd0}, 0xc0cb731a28, 0x1, 0xc07c2010a8, 0x519bbc, 0xc05edb8ba0, 0xc059788a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04fff5dd0}, 0x120, 0xc07c201230, 0x5edb8b40, 0x0, 0x660069, 0xc05edb8ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01edaa500, {0x63c85b8, 0xc04fff5dd0}, 0xc05edb8b40, {{{{0x0, 0x0}, {0xc01fe83440, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04fff5dd0}, 0x6352410, 0x17e2e3e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01e701220, {0x63c85b8, 0xc04fff5dd0}, {0xc05edb8a20, {0x1743011032cb16fd, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01e701220, {0x63c85b8, 0xc04fff5dd0}, 0xc071907360, 0xc07c2019f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04fff5dd0}, 0x660089, {0xc05edb8a20, {0x1743011032cb16fd, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc04fff5dd0}, 0xc04eeb9700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011032cb16fd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011032cb16fd, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04fff5c80}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04fff5c80}, {0xc07c2033f8, 0x660089}, 0xc07c203400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04fff5c80}, {0x4d939c}, 0xc04eeb9500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04fff5b30}, 0xc04eeb9500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}, 0xc0559030c0, 0xc0581884b0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}, 0x3, 0xc0c3355f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc07c203980}}}, {0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}, 0xc0559030c0, 0xc0c3355f00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04fff5b30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04fff5b30}, {0x40, 0xc07c203b70}, 0xc07c203ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04fff5b30}, {0x4e9d7a0, 0xc04eeb9500}, 0xc0559030c0, 0xc0581884b0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04fff5b30}, 0xc035a01380, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05c0e65a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05c0e65a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219795860 [runnable]: sync.runtime_SemacquireMutex(0x2a, 0x33, 0x57) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x40, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc091718cf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc081b92e00, {0xc0240fd9d0, 0x32, 0x6f}, {0xc05569a5a0, 0x32, 0x5f}, 0x1, 0x8c) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091718b88, {0xc0240fd9d0, 0x1, 0x5f}, {0xc05569a5a0, 0x2cf, 0xc07b0c6790}, 0x5b) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091718b88, {0xc0240fd9d0, 0x32, 0x6f}, {0xc05569a5a0, 0x32, 0x5f}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0917182c0, {0xc05569a5a0, 0x5f, 0x6f}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0917182c0, {0xc0240fd9d0, 0x5f, 0xc05569a5a0}, {0xc05569a5a0, 0xc073186000, 0x31}, 0x70) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091718000, {0xc05569a5a0, 0x32, 0x5f}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc06db5aea8, {{0xc073186000, 0x31, 0x70}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc077d4ea00, {{0xc070b16000, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc048a0b800, {0x63c85b8, 0xc0837a2240}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0837a2240}, {0x64a6560, 0xc077d4ea00}, {0xc070b16000, 0x646abf0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0837a2240}, {0x6466ed8, 0xc06db5ac00}, {0xc070b16000, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02c77f470}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc077208500, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc0201d6400, {0x63c85b8, 0xc0837a2240}, 0xc077208500, 0xc047b8c820) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0201d6400, {0x63c85b8, 0xc0837a2240}, 0xc077208500, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0837a21b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0837a21b0}, {0xc09365e8f0, 0x660089}, 0xc09365e8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0837a21b0}, {0xc00102e4e0}, 0xc077208400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0837a2150}, 0xc077208400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc062c8acf0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803095 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x69, 0xc06de1b050) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc06de1b0b8, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc091b425f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc054124000, {0xc0b9ee53e0, 0x2b, 0x60}, {0xc0c0a7df80, 0x2b, 0x6d}, 0x1, 0x25) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc091b42488, {0xc0b9ee53e0, 0x1, 0x6d}, {0xc0c0a7df80, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc091b42488, {0xc0b9ee53e0, 0x2b, 0x60}, {0xc0c0a7df80, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc091b41bc0, {0xc0c0a7df80, 0x6d, 0x60}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc091b41bc0, {0xc0b9ee53e0, 0x2b, 0xc0c0a7df80}, {0xc0c0a7df80, 0x2a, 0x30}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc091b41900, {0xc0c0a7df80, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc079a1f188, {{0xc0303695f0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc060cca800, {{0xc0303695f0, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc041c34400, {0x63c85b8, 0xc06489ac60}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc06489ac60}, {0x64a6560, 0xc060cca800}, {0xc0303695f0, 0xc00bc1fa40, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc06489ac60}, {0x6466ed8, 0xc079a1f180}, {0xc0303695f0, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc03c564bb8}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc03b7cec00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f4e800, {0x63c85b8, 0xc06489ac60}, 0xc03b7cec00, 0xc0b202a460) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f4e800, {0x63c85b8, 0xc06489ac60}, 0xc03b7cec00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06489abd0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06489abd0}, {0xc06de213f8, 0x660089}, 0xc06de21400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06489abd0}, {0x7fe34bbd6790}, 0xc03b7ce900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06489ab40}, 0xc03b7ce900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}, 0xc072c82ca0, 0xc028a600c0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}, 0x3, 0xc02846d040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06de21980}}}, {0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}, 0xc072c82ca0, 0xc02846d040) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06489ab40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06489ab40}, {0x40, 0xc06de21b70}, 0xc06de21ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06489ab40}, {0x4e9d7a0, 0xc03b7ce900}, 0xc072c82ca0, 0xc028a600c0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06489ab40}, 0xc06f983bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060c1de60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc060c1de60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812726 [runnable]: sync.runtime_SemacquireMutex(0x0, 0x0, 0x0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0xffff, 0xc07cbc23f0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc07c005b80, {0x4d866e, 0x91855c0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readIndex(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2244 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).init(0xc007a72000, 0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:266 +0x85 github.com/cockroachdb/pebble/sstable.(*Reader).NewIterWithBlockPropertyFilters(0xc07c005b80, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2165 +0x185 github.com/cockroachdb/pebble.(*tableCacheShard).newIters(0xc000a52140, 0xc0400ad198, 0xc0933f6438, 0x0, 0xc0002f1488) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:372 +0x27a github.com/cockroachdb/pebble.(*tableCacheContainer).newIters(0x116e025, 0xc0933f6550, 0x0, 0xc0933f6550) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:117 +0x45 github.com/cockroachdb/pebble.(*levelIter).loadFile(0xc0933f63e8, 0xc071571b80, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:348 +0x20f github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0933f63e8, {0xc023dab740, 0x18, 0x5d}, {0xc074ed66c0, 0x18, 0x58}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:425 +0xbf github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0933f51c0, {0xc074ed66c0, 0x0, 0xc0400ad4e0}, 0x121ea30, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0933f51c0, {0xc023dab740, 0x18, 0x58}, {0xc074ed66c0, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0933f4f00, {0xc074ed66c0, 0x18, 0x58}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc051db5b88, {{0xc08e777350, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0517d3000, {{0xc08e777350, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc03ee91000, {0x63c85b8, 0xc062cde810}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc062cde810}, {0x64a6560, 0xc0517d3000}, {0xc08e777350, 0xbcb5a4e00cbec29e, 0x18}, {0x1743011094713c1a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc062cde810}, {0x6466ed8, 0xc051db5b80}, {0xc08e777350, 0x17, 0x18}, {0x1743011094713c1a, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0252feee8}, {{0x1743011094713c1a, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743011094713c1a, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc050ef2900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b23ea11a, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d6400, {0x63c85b8, 0xc062cde810}, 0xc050ef2900, 0xc077bd6fa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d6400, {0x63c85b8, 0xc062cde810}, 0xc050ef2900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011094713c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011094713c1a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc062cde750}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cde750}, {0xc0400b33f8, 0x660089}, 0xc0400b3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc062cde750}, {0x61382d393235342d}, 0xc050ef2700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc062cde6c0}, 0xc050ef2700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}, 0xc06ac12ae0, 0xc041139fe0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}, 0x3, 0xc01c0bdc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0400b3980}}}, {0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}, 0xc06ac12ae0, 0xc01c0bdc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc062cde6c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc062cde6c0}, {0x40, 0xc0400b3b70}, 0xc0400b3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}, 0x4beac80, 0x7fe457d7f501) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc062cde6c0}, {0x4e9d7a0, 0xc050ef2700}, 0xc06ac12ae0, 0xc041139fe0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc062cde6c0}, 0xc049257560, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a59e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc0129a59e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801872 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc0c17c6a50}, {0xc05b672420, 0x2, 0xc00a817658}, {0x0, {0x1743010ff0fc7575, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc0c17c6a50}, 0xf825, 0xc05663f0e0, 0x2b, 0xc081f39010) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc0c17c6a50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219805297 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016c3b2c0, {0x63c85b8, 0xc077013b00}, 0xc06f264978, 0x1, 0x708017e10038c0, 0x37e11d600, 0xc05ef2a600, 0xc05ec33ec0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc077013b00}, 0x120, 0xc06e0cd230, 0x5ef2a5a0, 0xc010949860, 0x18, 0xc05ef2a600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016c3b2c0, {0x63c85b8, 0xc077013b00}, 0xc05ef2a5a0, {{{{0x0, 0x0}, {0xc036dd57a0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc077013b00}, 0x6352410, 0x17bb3a98) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce21e0, {0x63c85b8, 0xc077013b00}, {0xc05ef2a480, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce21e0, {0x63c85b8, 0xc077013b00}, 0xc076d89400, 0xc06e0cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc077013b00}, 0x660089, {0xc05ef2a480, {0x1743011043ff7686, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016d2a000, {0x63c85b8, 0xc077013b00}, 0xc0472aa300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043ff7686, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc077013a70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc077013a70}, {0xc06e0cf3f8, 0x660089}, 0xc06e0cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc077013a70}, {0x4677c5}, 0xc0472aa100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0770139e0}, 0xc0472aa100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}, 0xc06f2362c0, 0xc06f264900) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}, 0x3, 0xc077be34c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06e0cf980}}}, {0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}, 0xc06f2362c0, 0xc077be34c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0770139e0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0770139e0}, {0x40, 0xc06e0cfb70}, 0xc06e0cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}, 0x20301d, 0x20301d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0770139e0}, {0x4e9d7a0, 0xc0472aa100}, 0xc06f2362c0, 0xc06f264900) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0770139e0}, 0xc02b1d10e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05edb7d40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc05edb7d40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219809157 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).ResolveIntents(0xc0078dea50, {0x63c85b8, 0xc062c31440}, {0xc05c1fc840, 0x4, 0xc002f45e58}, {0x0, {0x1743010fea956608, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:895 +0x7e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents(0xc0078dea50, {0x63c85b8, 0xc062c31440}, 0x4ee, 0xc061970360, 0x20, 0xc059023d80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:771 +0x130 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).CleanupTxnIntentsAsync.func2({0x63c85b8, 0xc062c31440}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:577 +0xe5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808214 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc01eed7500}, 0xc02a4b0618, 0x1, 0x0, 0x1, 0xc079ab0840, 0xc0774c4600) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01eed7500}, 0x120, 0xc006b9a728, 0x79ab07e0, 0x0, 0x660069, 0xc079ab0840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc01eed7500}, 0xc079ab07e0, {{{{0x0, 0x0}, {0xc05c1ed8c0, 0x11d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01eed7500}, 0x6352410, 0x2ad6f208) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc01eed7500}, {0xc0790e4fc0, {0x174301105aab2da7, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc01eed7500}, 0xc021fc3b80, 0xc0966feee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01eed7500}, 0x660089, {0xc0790e4fc0, {0x174301105aab2da7, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc01eed7500}, 0xc078f0a200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105aab2da7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105aab2da7, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01eed7350}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01eed7350}, {0xc0036aa8f0, 0x660089}, 0xc0036aa8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01eed7350}, {0xc00102e4e0}, 0xc078f0a100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01eed7260}, 0xc078f0a100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07b5096b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801440 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0980a4548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0980a45b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc09254fbf0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc05812dc00, {0xc0ef182a00, 0x2b, 0x80}, {0xc0156e3c00, 0x2b, 0x6d}, 0x1, 0xfb) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc09254fa88, {0xc0ef182a00, 0x1, 0x6d}, {0xc0156e3c00, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09254fa88, {0xc0ef182a00, 0x2b, 0x80}, {0xc0156e3c00, 0x2b, 0x6d}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09254f1c0, {0xc0156e3c00, 0x6d, 0x80}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09254f1c0, {0xc0ef182a00, 0x2b, 0xc0156e3c00}, {0xc0156e3c00, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09254ef00, {0xc0156e3c00, 0x2b, 0x6d}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc04a893608, {{0xc047ee72c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc04be2c900, {{0xc047ee72c0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0cd8eb800, {0x63c85b8, 0xc0750174a0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0750174a0}, {0x64a6560, 0xc04be2c900}, {0xc047ee72c0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0750174a0}, {0x6466ed8, 0xc04a893600}, {0xc047ee72c0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0198ab6f8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc04b2a1a00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020d8a000, {0x63c85b8, 0xc0750174a0}, 0xc04b2a1a00, 0xc05002bae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020d8a000, {0x63c85b8, 0xc0750174a0}, 0xc04b2a1a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc075017410}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc075017410}, {0xc0ad8388f0, 0x660089}, 0xc0ad8388f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc075017410}, {0xc00102e4e0}, 0xc04b2a1900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0750173b0}, 0xc04b2a1900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02ef02a20}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803512 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc00d364750}, 0xc047405620, 0x1, 0xc0678d70a8, 0x519bbc, 0xc03c4023c0, 0xc039503c80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc00d364750}, 0x120, 0xc0678d7230, 0x3c402360, 0x0, 0x660069, 0xc03c4023c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc00d364750}, 0xc03c402360, {{{{0x0, 0x0}, {0xc048a1bb00, 0xe0}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc00d364750}, 0x6352410, 0x45bf43a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc00d364750}, {0xc03c402240, {0x174301102d052c72, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc00d364750}, 0xc02eccc6e0, 0xc0678d79f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc00d364750}, 0x660089, {0xc03c402240, {0x174301102d052c72, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc00d364750}, 0xc038cec100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d052c72, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d052c72, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc00d364690}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00d364690}, {0xc0678d93f8, 0x660089}, 0xc0678d9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc00d364690}, {0x7fe3f1f037f0}, 0xc00d051f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc00d364570}, 0xc00d051f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}, 0xc035050820, 0xc0379d1680) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}, 0x3, 0xc020de7340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0678d9980}}}, {0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}, 0xc035050820, 0xc020de7340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc00d364570}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc00d364570}, {0x40, 0xc0678d9b70}, 0xc0678d9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc00d364570}, {0x4e9d7a0, 0xc00d051f00}, 0xc035050820, 0xc0379d1680) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc00d364570}, 0xc022266000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03c402120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03c402120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811266 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc072608ed0}, 0xc026b79bc0, 0x1, 0xc0063b10a8, 0x519bbc, 0xc08a7f83c0, 0xc033cf5020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc072608ed0}, 0x120, 0xc0063b1230, 0x8a7f8360, 0x0, 0x660069, 0xc08a7f83c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc072608ed0}, 0xc08a7f8360, {{{{0x0, 0x0}, {0xc09c0bd440, 0x108}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc072608ed0}, 0x6352410, 0x202c44b0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc072608ed0}, {0xc08a7f8240, {0x1743011072ddd270, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc072608ed0}, 0xc06ab62140, 0xc0063b19f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc072608ed0}, 0x660089, {0xc08a7f8240, {0x1743011072ddd270, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc072608ed0}, 0xc04de36e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011072ddd270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011072ddd270, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc072608e40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072608e40}, {0xc0063b33f8, 0x660089}, 0xc0063b3400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc072608e40}, {0x0}, 0xc04de36c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc072608db0}, 0xc04de36c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}, 0xc0148defe0, 0xc03a289d40) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}, 0x3, 0xc074716280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0063b3980}}}, {0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}, 0xc0148defe0, 0xc074716280) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc072608db0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc072608db0}, {0x40, 0xc0063b3b70}, 0xc0063b3ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc072608db0}, {0x4e9d7a0, 0xc04de36c00}, 0xc0148defe0, 0xc03a289d40) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc072608db0}, 0xc03ffc0a20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08a7f8120, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc08a7f8120, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810257 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01edcb400, {0x63c85b8, 0xc021e00e70}, 0xc063592200, 0xc0472090e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01edcb400, {0x63c85b8, 0xc021e00e70}, 0xc063592200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106d0fcbe0, 0x0, 0x0}, 0xc04e1e8b90, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106d0fcbe0, 0x0, 0x0}, 0xc04e1e8b90, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x3, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc021e00c90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021e00c90}, {0xc0294773f8, 0x660089}, 0xc029477400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc021e00c90}, {0xc0015221e0}, 0xc063592000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc021e00ba0}, 0xc063592000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}, 0xc03cd879e0, 0xc0039f2a68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}, 0x3, 0xc050e44e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc029477980}}}, {0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}, 0xc03cd879e0, 0xc050e44e40) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc021e00ba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc021e00ba0}, {0x40, 0xc029477b70}, 0xc029477ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}, 0x4beac80, 0x7fe457d82901) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc021e00ba0}, {0x4e9d7a0, 0xc063592000}, 0xc03cd879e0, 0xc0039f2a68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc021e00ba0}, 0xc0cd486e40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc067423320, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc067423320, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811217 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc0413e4cc0}, 0xc03c991f50, 0x1, 0x0, 0x1, 0xc016b70ba0, 0xc01855f380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0413e4cc0}, 0x120, 0xc027e48728, 0x16b70b40, 0x0, 0x660069, 0xc016b70ba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc0413e4cc0}, 0xc016b70b40, {{{{0x0, 0x0}, {0xc03eb87200, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0413e4cc0}, 0x6352410, 0x6847e060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc0413e4cc0}, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc0413e4cc0}, 0xc079989a40, 0xc089d54ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0413e4cc0}, 0x660089, {0xc00ec82a20, {0x174301106bb1c902, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc0413e4cc0}, 0xc06f379200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bb1c902, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0413e4c30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0413e4c30}, {0xc0936968f0, 0x660089}, 0xc0936968f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0413e4c30}, {0xc00102e4e0}, 0xc06f379100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0413e4ba0}, 0xc06f379100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc044fe6e40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806372 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc07bf2dc20}, 0xc09b7d9500, 0xc0239eafa0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc07bf2dc20}, 0xc09b7d9500, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104e36cc5f, 0x0, 0x0}, 0xc077105320, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104e36cc5f, 0x0, 0x0}, 0xc077105320, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07bf2db90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bf2db90}, {0xc00b0f93f8, 0x660089}, 0xc00b0f9400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07bf2db90}, {0x556e67312b795a53}, 0xc09b7d9100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07bf2db00}, 0xc09b7d9100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}, 0xc07a55c200, 0xc08e6f1f68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}, 0x3, 0xc07aab4c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00b0f9980}}}, {0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}, 0xc07a55c200, 0xc07aab4c00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07bf2db00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07bf2db00}, {0x40, 0xc00b0f9b70}, 0xc00b0f9ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07bf2db00}, {0x4e9d7a0, 0xc09b7d9100}, 0xc07a55c200, 0xc08e6f1f68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07bf2db00}, 0xc048be6d20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc046064360, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc046064360, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219582941 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0571b1e60) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc05f987b80, {0x4d83cc0, 0xc0d0c00e80}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xf5) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc0571b1c20, 0xc0548690b8, 0xc054869088) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc0571b1c20, {0x4d83cc0, 0xc0d0c00e80}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc0461a4270}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc06b6e2300}, {0x4d83cc0, 0xc0d0c00e80}, 0x2, {0xc00c7a1180, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc0461a4270}, {0x4f6406d, 0x7fe457d81688}, {0x4e9d7a0, 0xc06b6e2300}, {0x4d83cc0, 0xc0d0c00e80}, 0x7c3750d26a34eee7, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x37b05080d6b6194e, {0x63c85b8, 0xc0461a4270}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc06b6e2300}, {0x4d83cc0, 0xc0d0c00e80}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc055bf2d90, {0x63c85b8, 0xc0461a4270}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc055bf2d90}}, {0x63c85b8, 0xc0461a4270}, 0xc06b6e2300, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1d, 0x1d, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc0a9912000, {0x63c85b8, 0xc0773f8390}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc0a9912000, {0x63c85b8, 0xc0773f8390}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc0741691d0, {0x63c85b8, 0xc0773f8390}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc055a04e00, {0x63c85b8, 0xc0773f8390}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc00d7f22c0) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:264 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc043f7f980) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc0391f0780) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc042d50f00) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc070829970, {0x63c8510, 0xc043f7f8c0}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0dbcf9450, {0x63c8510, 0xc043f7f8c0}, 0xc045f52210) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc043f7f700}, 0xc02553c2a0, 0xc045f52210, 0xc06b559000, 0xc00151e700, 0xc05648bce8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc043f7f700}, 0xc05648b9f0, 0xc02553c2a0, 0xc0b31b7bc0, {{0x63ca960, 0xc06cb26be0}, 0x0}, 0xc00151e700) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc05648b300, {0x63c8510, 0xc043f7f700}, 0xc05648b9f0, 0xc043f7f700, {0x7fe42b790cf0, 0xc0b31b7bc0}, 0x0, 0xc0391f0658) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc05648b300, {0x63c8510, 0xc043f7f700}, 0xc05648b9f0, {0x7fe42b790cf0, 0xc0b31b7bc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc05648b300, {0x63c85b8, 0xc02272c810}, {{0x63f8d78, 0xc02df30a00}, {0xc061712010, 0x174}, 0x1, 0x1}, 0xc06653e5a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc05648b300, {0x63c85b8, 0xc02272c810}, {{0x63f8d78, 0xc02df30a00}, {0xc061712010, 0x174}, 0x1, 0x1}, 0xc06653e5a0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc05648b300, {0x63c85b8, 0xc02272c810}, {0xc06653e5a0, {0xc04c6818b0, 0x1, 0x1}, {0xc08be8c1e0, 0x12, 0x12}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc0a34ab67e, 0x0}, 0x0, {0x1cbeb668, 0xedb7a8350, 0x0}, 0x1}, 0xc05648b300, 0xc05486f980, 0xc05486f970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc05648b300) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc05648b300, {0x63c8510, 0xc03c5e4740}, 0xc055dc3dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03c4e4c80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03c5e4740}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc03c4e4c80) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219808835 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e0a7040, {0x63c85b8, 0xc0708150e0}, 0xc043919428, 0x1, 0xc00806b0a8, 0x519bbc, 0xc02eb81e60, 0xc074e79500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0708150e0}, 0x1e0, 0xc00806b230, 0x2eb81e00, 0x0, 0x660069, 0xc02eb81e60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e0a7040, {0x63c85b8, 0xc0708150e0}, 0xc02eb81e00, {{{{0x0, 0x0}, {0xc0a8bcf200, 0x59}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0708150e0}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e0c2370, {0x63c85b8, 0xc0708150e0}, {0xc02ddee5a0, {0x174301105fd3e432, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e0c2370, {0x63c85b8, 0xc0708150e0}, 0xc071f3b900, 0xc00806b9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0708150e0}, 0x660089, {0xc02ddee5a0, {0x174301105fd3e432, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d4c00, {0x63c85b8, 0xc0708150e0}, 0xc04854a200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105fd3e432, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105fd3e432, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc070815050}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070815050}, {0xc00806d3f8, 0x660089}, 0xc00806d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc070815050}, {0x7fe3c2e70b20}, 0xc04854a000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc070814f30}, 0xc04854a000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}, 0xc029a8e880, 0xc043919710) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}, 0x3, 0xc09f0c13c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc00806d980}}}, {0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}, 0xc029a8e880, 0xc09f0c13c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc070814f30}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc070814f30}, {0x40, 0xc00806db70}, 0xc00806dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc070814f30}, {0x4e9d7a0, 0xc04854a000}, 0xc029a8e880, 0xc043919710) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc070814f30}, 0xc020e3fd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc02ddee480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc02ddee480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219812740 [runnable]: google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219781602 [IO wait]: internal/poll.runtime_pollWait(0x7fdc6af29438, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc051a7a580, 0xc051905800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc051a7a580, {0xc051905800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc051a7a580, {0xc051905800, 0x735, 0xc05196e000}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc01baba2d0, {0xc051905800, 0x7eda79, 0xc051905805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0b5fcad80, {0xc051905800, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc0342fd458, {0xc051905800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc04ba4ccf8, {0x631b900, 0xc0342fd458}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc04ba4ca80, {0x7fe42ab59310, 0xc0b5fcad80}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc04ba4ca80, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc04ba4ca80, {0xc0ac1fe000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc026ceb860, {0xc0ac1fe000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc03424b588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc03424b588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc03424bb60, {0x634aac0, 0xc03424b588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc03424b500, 0xc00f653b70, 0xc09753742b, 0xc09753742d, {0x635bec0, 0xc098138990}, 0xc000d6ae00, 0xc03eb2ee28, 0xc097537429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc03424b500, {0x63c85b8, 0xc01040ab10}, 0xc00f653b60, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc023c4a037, 0x8}}, 0x0, 0xc01040a4b0, 0xc01040a510, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc01040ab10}, {0x643d810, 0xc0b5fcad80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0411f7ce0}, {0x643d810, 0xc0b5fcad80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0411f7ce0}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810407 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc07995adb0}, 0xc027348af8, 0x1, 0xc013b9d0a8, 0x519bbc, 0xc075b964e0, 0xc059ee6960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc04d7faa80, {0x63c85b8, 0xc07995adb0}, 0xc036cb6900, 0xc013b9d1a8, 0x75b96540, 0x0, 0x2, 0xc075b964e0, 0x51c47e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc07995adb0}, 0xc075b96480, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc045559d40, 0xa8}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:485 +0x425 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x63c85b8, {0x63c85b8, 0xc07995adb0}, 0x63523b0, 0x55bf2490) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc07995adb0}, {0x0, {0x174301106eb75d62, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc07995adb0}, 0xc03b364be0, 0xc013b9d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07995adb0}, 0x660089, {0x0, {0x174301106eb75d62, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07995adb0}, 0xc086e93200, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106eb75d62, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106eb75d62, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x174301106eb75d62, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07995ad20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07995ad20}, {0xc013b9f3f8, 0x660089}, 0xc013b9f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07995ad20}, {0xc0015221e0}, 0xc086e92f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07995ac90}, 0xc086e92f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}, 0xc06734c460, 0xc01c4383a8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}, 0x3, 0xc015d27100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc013b9f980}}}, {0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}, 0xc06734c460, 0xc015d27100) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07995ac90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07995ac90}, {0x40, 0xc013b9fb70}, 0xc013b9fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07995ac90}, {0x4e9d7a0, 0xc086e92f00}, 0xc06734c460, 0xc01c4383a8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07995ac90}, 0xc0194c84e0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc054b28ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc054b28ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219730597 [IO wait]: internal/poll.runtime_pollWait(0x7fe3ac0a2340, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc06484cc80, 0xc0436c7000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc06484cc80, {0xc0436c7000, 0x2b83, 0x2b83}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc06484cc80, {0xc0436c7000, 0x2b83, 0xc05a45ba40}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc0553a7740, {0xc0436c7000, 0x7eda79, 0xc0436c7005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc06e27b140, {0xc0436c7000, 0x0, 0x2b83}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc038b13c80, {0xc0436c7000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0337a13f8, {0x631b900, 0xc038b13c80}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0337a1180, {0x7fe42ab59310, 0xc06e27b140}, 0x2b83) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0337a1180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0337a1180, {0xc08462c000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc02b957d40, {0xc08462c000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc05fb3cb08) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc05fb3cb08) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc05fb3d0e0, {0x634aac0, 0xc05fb3cb08}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc05fb3ca80, 0xc056184740, 0xc09755342b, 0xc09755342d, {0x635bec0, 0xc05a92af30}, 0xc000d6ae00, 0xc045fa2088, 0xc097553429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc05fb3ca80, {0x63c85b8, 0xc06d3d3500}, 0xc056184730, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc060ade009, 0x8}}, 0x0, 0xc06d3d34a0, 0xc06d3d34d0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc06d3d3500}, {0x643d810, 0xc06e27b140}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc06e0bae70}, {0x643d810, 0xc06e27b140}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc06e0bae70}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806505 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0219a0500, {0x63c85b8, 0xc08643e240}, 0xc090f95530, 0x1, 0xc00491f0a8, 0x519bbc, 0xc0912fb6e0, 0xc0c4e04ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc08643e240}, 0x1e0, 0xc00491f230, 0x912fb680, 0x0, 0x660069, 0xc0912fb6e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0219a0500, {0x63c85b8, 0xc08643e240}, 0xc0912fb680, {{{{0x0, 0x0}, {0xc05a482b40, 0x9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x10000, {0x63c85b8, 0xc08643e240}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01da15680, {0x63c85b8, 0xc08643e240}, {0xc055da6fc0, {0x174301104dfd0e9c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01da15680, {0x63c85b8, 0xc08643e240}, 0xc01d7d2820, 0xc00491f9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08643e240}, 0x660089, {0xc055da6fc0, {0x174301104dfd0e9c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0219b8000, {0x63c85b8, 0xc08643e240}, 0xc0676a7f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104dfd0e9c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104dfd0e9c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08643e1b0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08643e1b0}, {0xc0049213f8, 0x660089}, 0xc004921400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08643e1b0}, {0xc0015221e0}, 0xc0676a7d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08643e120}, 0xc0676a7d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}, 0xc08a538ba0, 0xc090f95f20) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}, 0x3, 0xc0824cc440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004921980}}}, {0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}, 0xc08a538ba0, 0xc0824cc440) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc08643e120}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08643e120}, {0x40, 0xc004921b70}, 0xc004921ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}, 0x4beac80, 0x7fe457d80d01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc08643e120}, {0x4e9d7a0, 0xc0676a7d00}, 0xc08a538ba0, 0xc090f95f20) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc08643e120}, 0xc093023680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc055da6ea0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc055da6ea0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219793544 [runnable]: sync.runtime_SemacquireMutex(0xc005033198, 0x8f, 0x9bb12c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc005033040) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2, 0x3dfafcf, 0xc005033068) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc079ba6000, {0xffffffffffffff15, 0xc0050331c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc079ba6000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc079ba6000, {0xc00a67d860, 0x32, 0x93}, {0xc02c46c500, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc07813ad98, {0xc00a67d860, 0x32, 0x93}, {0xc02c46c500, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0781396c0, {0xc02c46c500, 0x74, 0x93}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0781396c0, {0xc00a67d860, 0x74, 0xc02c46c500}, {0xc02c46c500, 0xc01f2ead20, 0x31}, 0xa0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc078139400, {0xc02c46c500, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0746122a8, {{0xc01f2ead20, 0x31, 0xa0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc032a1e700, {{0xc05ee37470, 0x2a, 0x30}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0c11a7400, {0x63c85b8, 0xc0388e4f60}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0388e4f60}, {0x64a6560, 0xc032a1e700}, {0xc05ee37470, 0xc036f56680, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0388e4f60}, {0x6466ed8, 0xc074612000}, {0xc05ee37470, 0x2a, 0x30}, {0x174300f626d67c30, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0232395d8}, {{0x174300f626d67c30, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06f31d300, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x174300f626d67c30, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020a33c00, {0x63c85b8, 0xc0388e4f60}, 0xc06f31d300, 0xc066746960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020a33c00, {0x63c85b8, 0xc0388e4f60}, 0xc06f31d300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174300f626d67c30, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0xd, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0388e4ed0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0388e4ed0}, {0xc0050393f8, 0x660089}, 0xc005039400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0388e4ed0}, {0xc019c81080}, 0xc06f31cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0388e4e40}, 0xc06f31cf00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}, 0xc0557192e0, 0xc03e577b90) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}, 0x3, 0xc011288bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc005039980}}}, {0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}, 0xc0557192e0, 0xc011288bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0388e4e40}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0388e4e40}, {0x40, 0xc005039b70}, 0xc005039ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0388e4e40}, {0x4e9d7a0, 0xc06f31cf00}, 0xc0557192e0, 0xc03e577b90) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0388e4e40}, 0xc0cc8f3bc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01221b0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc01221b0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219790150 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01d882500, {0x63c85b8, 0xc0327d2750}, 0xc044339f98, 0x1, 0xc07a16f2b0, 0x519bbc, 0xc043aa4240, 0xc04b200fe0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc08a154280, {0x63c85b8, 0xc0327d2750}, 0xc03a500120, 0xc07a16f3b0, 0x21, 0x40, 0x0, 0xc043aa4240, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01d882500, {0x63c85b8, 0xc0327d2750}, 0xc043aa41e0, {{{{0xc044876480, 0x10}, {0xc012143440, 0x1f}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x40, {0x63c85b8, 0xc0327d2750}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01d7f4f00, {0x63c85b8, 0xc0327d2750}, {0xc062fb6b40, {0x174301106c04a211, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01d7f4f00, {0x63c85b8, 0xc0327d2750}, 0xc0489f4280, 0xc07a16fbf8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0327d2750}, 0x660089, {0xc062fb6b40, {0x174301106c04a211, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d898800, {0x63c85b8, 0xc0327d2750}, 0xc041cc4000, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106c04a211, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106c04a211, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0327d26c0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0327d26c0}, {0xc07a171600, 0x660089}, 0xc07a171608) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0327d26c0}, {0xc00102e4e0}, 0xc04bcdff00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0327d2600}, 0xc04bcdff00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x1, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1374 +0x10c5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.sendAndFill({0x63c8510, 0xc06b531300}, 0xe, 0xc074f20b00) github.com/cockroachdb/cockroach/pkg/kv/db.go:830 +0xfc github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc06f4b3810, {0x63c8510, 0xc06b531300}, 0x0) github.com/cockroachdb/cockroach/pkg/kv/txn.go:696 +0x74 github.com/cockroachdb/cockroach/pkg/sql.(*tableWriterBase).finalize(0xc07a97a728, {0x63c8510, 0xc06b531300}) github.com/cockroachdb/cockroach/pkg/sql/tablewriter.go:225 +0xb6 github.com/cockroachdb/cockroach/pkg/sql.(*updateNode).BatchedNext(0xc07a97a700, {{0x63c8510, 0xc06b531300}, 0xc0529d3180, 0xc0511a92f0}) github.com/cockroachdb/cockroach/pkg/sql/update.go:196 +0x210 github.com/cockroachdb/cockroach/pkg/sql.(*rowCountNode).startExec(0xc0298d6d80, {{0x63c8510, 0xc06b531300}, 0xc0529d3180, 0xc0511a92f0}) github.com/cockroachdb/cockroach/pkg/sql/plan_batch.go:173 +0xce github.com/cockroachdb/cockroach/pkg/sql.startExec.func2({0xc07a175930, 0x63cb098}, {0x63caa08, 0xc0298d6d80}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:516 +0x147 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal.func1() github.com/cockroachdb/cockroach/pkg/sql/walk.go:112 +0x3e github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visitInternal(0xc07a175930, {0x63caa08, 0xc0298d6d80}, {0x4ed688c, 0x5}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:297 +0x14c9 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc07a175930, {0x63caa08, 0xc0298d6d80}) github.com/cockroachdb/cockroach/pkg/sql/walk.go:79 +0xf7 github.com/cockroachdb/cockroach/pkg/sql.walkPlan(...) github.com/cockroachdb/cockroach/pkg/sql/walk.go:43 github.com/cockroachdb/cockroach/pkg/sql.startExec({{0x63c8510, 0xc06b531300}, 0xc0529d3180, 0xc0511a92f0}, {0x63caa08, 0xc0298d6d80}) github.com/cockroachdb/cockroach/pkg/sql/plan.go:519 +0x125 github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Start(0xc07daaf100, {0x63c8510, 0xc06b531300}) github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:146 +0x87 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start.func1() github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:120 +0x34 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x200000000) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Start(0xc0c9892360, {0x63c8510, 0xc06b531300}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:119 +0x6b github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc0c9892360, {0x63c8510, 0xc06b531300}) github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:753 +0x3a github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc0b578d380, {0x63c8510, 0xc06b531300}, 0xc0c9892360) github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:469 +0x258 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc0511a95e8, {0x63c8510, 0xc06b531300}, 0xc06f4b3810) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:303 +0x205 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc06b530f80}, 0xc0bf1eb6c0, 0xc06f4b3810, 0xc059103d80, 0xc0c87b3180, 0xc0511a95e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc06b530f80}, 0xc0511a92f0, 0xc0bf1eb6c0, 0xc0ccc846c0, {{0x63caa08, 0xc0298d6d80}, 0x0}, 0xc0c87b3180) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0511a8c00, {0x63c8510, 0xc06b530f80}, 0xc0511a92f0, 0xc06b530f80, {0x7fe42b790cf0, 0xc0ccc846c0}, 0x0, 0xc0b96427d8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0511a8c00, {0x63c8510, 0xc06b530f80}, 0xc0511a92f0, {0x7fe42b790cf0, 0xc0ccc846c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0511a8c00, {0x63c85b8, 0xc024f2f290}, {{0x63f9d78, 0xc068110e00}, {0xc081df8c80, 0x95}, 0x3, 0x1}, 0xc00ef31d40, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0511a8c00, {0x63c85b8, 0xc024f2f290}, {{0x63f9d78, 0xc068110e00}, {0xc081df8c80, 0x95}, 0x3, 0x1}, 0xc00ef31d40, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0511a8c00, {0x63c85b8, 0xc024f2f290}, {0xc00ef31d40, {0xc07a57e4b0, 0x3, 0x3}, {0x9bf64bc, 0x1, 0x1}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc09c204002, 0x0}, 0x0, {0x153c7325, 0xedb7a8350, 0x0}, 0x1}, 0xc0511a8c00, 0xc07a177980, 0xc07a177970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0511a8c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0511a8c00, {0x63c8510, 0xc00a6e4700}, 0xc057c77dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07092f570) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc00a6e4700}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc07092f570) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812563 [runnable]: syscall.Syscall6(0x11, 0x72a, 0x7fe36b251820, 0x35c6, 0x567ea5a, 0x0, 0x0) GOROOT/src/syscall/asm_linux_amd64.s:43 +0x5 syscall.Pread(0x0, {0x7fe36b251820, 0x0, 0xc043b6ede0}, 0x0) GOROOT/src/syscall/zsyscall_linux_amd64.go:1212 +0x58 internal/poll.(*FD).Pread(0xc061b5da00, {0x7fe36b251820, 0x4ba25b, 0x35c6}, 0x200000003) GOROOT/src/internal/poll/fd_unix.go:193 +0xf9 os.(*File).pread(...) GOROOT/src/os/file_posix.go:41 os.(*File).ReadAt(0xc0146f1048, {0x7fe36b251820, 0xc043b6eec8, 0x1184134}, 0x35e6) GOROOT/src/os/file.go:137 +0x191 github.com/cockroachdb/pebble/vfs.enospcFile.ReadAt(...) github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_full.go:361 github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc015e63b80, {0x567ea5a, 0x35c1}, 0x0, 0xc0871f2b78) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2339 +0x3f2 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0871f2900, {0xffffffffffffff15, 0xc043b6f0c8}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).loadBlock(0xc0871f2900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:380 +0x22a github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper(0xc0871f2900, {0xc0977fcf00, 0x18, 0x19}, 0x13, 0x53) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:580 +0x3ac github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc0871f2900, {0xc04ff144c8, 0x18, 0x19}, {0xc0977fcf00, 0x0, 0xc0871f2900}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:665 +0x2ec github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0871f2900, {0xc04ff144c8, 0x18, 0x18}, {0xc0977fcf00, 0x18, 0x19}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1417 +0x54c github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc092f8a398, {0xc04ff144c8, 0x18, 0x18}, {0xc0977fcf00, 0x18, 0x19}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc092f88cc0, {0xc0977fcf00, 0x0, 0xc043b6f4e0}, 0x121ea30, 0x30) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc092f88cc0, {0xc04ff144c8, 0x18, 0x19}, {0xc0977fcf00, 0x17, 0x18}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc092f88a00, {0xc0977fcf00, 0x18, 0x19}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc052d3a108, {{0xc0578ee6c0, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc0a0752c00, {{0xc0578ee6c0, 0x17, 0x18}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0450ccc00, {0x63c85b8, 0xc056615e00}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc056615e00}, {0x64a6560, 0xc0a0752c00}, {0xc0578ee6c0, 0x0, 0x18}, {0x17430110959fbec4, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc056615e00}, {0x6466ed8, 0xc052d3a100}, {0xc0578ee6c0, 0x17, 0x18}, {0x17430110959fbec4, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc03f1469a8}, {{0x17430110959fbec4, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x17430110959fbec4, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc046449900, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x17430110b36d23c4, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc02e0d5800, {0x63c85b8, 0xc056615e00}, 0xc046449900, 0xc070c48dc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e0d5800, {0x63c85b8, 0xc056615e00}, 0xc046449900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110959fbec4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110959fbec4, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x15, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc056615d70}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056615d70}, {0xc043b753f8, 0x660089}, 0xc043b75400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc056615d70}, {0x3}, 0xc046449700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc056615cb0}, 0xc046449700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}, 0xc076de60a0, 0xc044abe258) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}, 0x3, 0xc071e55a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc043b75980}}}, {0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}, 0xc076de60a0, 0xc071e55a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc056615cb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc056615cb0}, {0x40, 0xc043b75b70}, 0xc043b75ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc056615cb0}, {0x4e9d7a0, 0xc046449700}, 0xc076de60a0, 0xc044abe258) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc056615cb0}, 0xc058eb0420, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea3680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc032ea3680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810059 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc05ca98dc0, {0x63c85b8, 0xc078773f50}, 0xc01576e6d8, 0x1, 0xc01a6af670, 0x519bbc, 0xc07933e180, 0xc09cbd9860) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc03ce91c20, {0x63c85b8, 0xc078773f50}, 0xc07933e180, 0xc01a6af770, 0x53, 0x80, 0x5, 0xc07933e180, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc05ca98dc0, {0x63c85b8, 0xc078773f50}, 0xc07933e000, {{{{0xc064c9f680, 0x2}, {0xc019e1a120, 0x2}}, {{0xc05b6cb320, 0xa}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc078773f50}, 0x63521a0, 0x4b1bd648) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc045008410, {0x63c85b8, 0xc078773f50}, {0xc066c1ad80, {0x174301100ae19db9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc045008410, {0x63c85b8, 0xc078773f50}, 0xc066330780, 0xc01a6affb8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc078773f50}, 0x660089, {0xc066c1ad80, {0x174301100ae19db9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc06ffeec00, {0x63c85b8, 0xc078773f50}, 0xc048877b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301100ae19db9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301100ae19db9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc078773ec0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc078773ec0}, {0xc0d1bd59c0, 0x660089}, 0xc0d1bd59c8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc078773ec0}, {0xc00102e4e0}, 0xc048877a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc078773e60}, 0xc048877a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.makeTxnCommitExplicitLocked({0x63c85b8, 0xc045e7c030}, {0x7fe422a51230, 0xc07d859c00}, 0xc044fae6c0, {0xc0ad36f1a0, 0x1, 0x2}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:509 +0x24d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).makeTxnCommitExplicitAsync.func1({0x63c85b8, 0xc045e7c030}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:486 +0x114 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801254 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafe798b8, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc064a96580, 0xc033768000, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc064a96580, {0xc033768000, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc064a96580, {0xc033768000, 0x735, 0xc055f31400}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc03fdab648, {0xc033768000, 0x7eda79, 0xc033768005}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0705a5020, {0xc033768000, 0x0, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc02f753398, {0xc033768000, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc00d7bc5f8, {0x631b900, 0xc02f753398}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc00d7bc380, {0x7fe42ab59310, 0xc0705a5020}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc00d7bc380, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc00d7bc380, {0xc001a05000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01d2a5d40, {0xc001a05000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc099ebd588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc099ebd588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc099ebdb60, {0x634aac0, 0xc099ebd588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc099ebd500, 0xc073c92950, 0xc00696542b, 0xc00696542d, {0x635bec0, 0xc071b105a0}, 0xc000d6ae00, 0xc0978aadc8, 0xc006965429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc099ebd500, {0x63c85b8, 0xc08f6212c0}, 0xc073c92940, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc098df703e, 0x8}}, 0x0, 0xc08f621260, 0xc08f621290, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc08f6212c0}, {0x643d810, 0xc0705a5020}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc076804420}, {0x643d810, 0xc0705a5020}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc076804420}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812019 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0828cddd0}, 0xc0cc964a98, 0x1, 0x70801819d2d565, 0x37e11d600, 0xc0709051a0, 0xc0338b9560) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc06c00a4b0, {0x63c85b8, 0xc0828cddd0}, 0xc0bfb54480, 0xc004e091a8, 0x709051a0, 0x0, 0xffffffffffffffff, 0xc0709051a0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0828cddd0}, 0xc0709050e0, {{{{0xc018f21d40, 0x3c}, {0xc0a626e000, 0x66}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x30, {0x63c85b8, 0xc0828cddd0}, 0x0, 0x3) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0828cddd0}, {0xc03be598c0, {0x174301107d0c1b05, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0828cddd0}, 0xc048adfea0, 0xc004e099f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0828cddd0}, 0x660089, {0xc03be598c0, {0x174301107d0c1b05, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0828cddd0}, 0xc0827a5b00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107d0c1b05, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107d0c1b05, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0828cdd40}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0828cdd40}, {0xc004e0b3f8, 0x660089}, 0xc004e0b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0828cdd40}, {0x7fe429237f70}, 0xc0827a5900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0828cdcb0}, 0xc0827a5900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}, 0xc06b2354a0, 0xc0cc964a68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}, 0x3, 0xc063be4580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc004e0b980}}}, {0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}, 0xc06b2354a0, 0xc063be4580) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0828cdcb0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0828cdcb0}, {0x40, 0xc004e0bb70}, 0xc004e0bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0828cdcb0}, {0x4e9d7a0, 0xc0827a5900}, 0xc06b2354a0, 0xc0cc964a68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0828cdcb0}, 0xc04f98fec0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03be59680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc03be59680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802888 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1275 +0x11b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1440 +0x1479 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x63c85b8, 0xc06b7d76b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:991 +0x177 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802190 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc0980b4548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc0980b45b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc09273c0f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc05ab16700, {0xc022121480, 0x2b, 0x39}, {0xc05520ac90, 0x2b, 0x2b}, 0x1, 0xc0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc09273bf88, {0xc022121480, 0x1, 0x2b}, {0xc05520ac90, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09273bf88, {0xc022121480, 0x2b, 0x39}, {0xc05520ac90, 0x2b, 0x2b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09273b6c0, {0xc05520ac90, 0x2b, 0x39}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09273b6c0, {0xc022121480, 0x2b, 0xc05520ac90}, {0xc05520ac90, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09273b400, {0xc05520ac90, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc061c26b08, {{0xc062ce37d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc06adbe700, {{0xc062ce37d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04c556400, {0x63c85b8, 0xc0d04e61e0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0d04e61e0}, {0x64a6560, 0xc06adbe700}, {0xc062ce37d0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0d04e61e0}, {0x6466ed8, 0xc061c26b00}, {0xc062ce37d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc06af88870}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc052b22600, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020ee0800, {0x63c85b8, 0xc0d04e61e0}, 0xc052b22600, 0xc02f38fae0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020ee0800, {0x63c85b8, 0xc0d04e61e0}, 0xc052b22600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0d04e6150}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0d04e6150}, {0xc07d2548f0, 0x660089}, 0xc07d2548f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0d04e6150}, {0xc00102e4e0}, 0xc052b22500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0d04e60f0}, 0xc052b22500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220480}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801361 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc04e55a360}, 0xc025d1cc00, 0x1, 0xc04bcad0a8, 0x519bbc, 0xc01cf23260, 0xc055d4b5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04e55a360}, 0x120, 0xc04bcad230, 0x1cf23200, 0x0, 0x660069, 0xc01cf23260, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc04e55a360}, 0xc01cf23200, {{{{0x0, 0x0}, {0xc05bd28fc0, 0xe9}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04e55a360}, 0x6352410, 0x6541ed90) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc04e55a360}, {0xc01cf22d80, {0x174301101c01e56f, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc04e55a360}, 0xc05292ee60, 0xc04bcad9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04e55a360}, 0x660089, {0xc01cf22d80, {0x174301101c01e56f, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04e55a360}, 0xc065171c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301101c01e56f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301101c01e56f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e55a2d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e55a2d0}, {0xc04bcaf3f8, 0x660089}, 0xc04bcaf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e55a2d0}, {0xc0869ad2d8}, 0xc065171a00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e55a240}, 0xc065171a00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}, 0xc060421e00, 0xc071bdb968) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}, 0x3, 0xc011d07b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc04bcaf980}}}, {0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}, 0xc060421e00, 0xc011d07b00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e55a240}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e55a240}, {0x40, 0xc04bcafb70}, 0xc04bcafab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}, 0x4beac80, 0x7fe457d7f101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e55a240}, {0x4e9d7a0, 0xc065171a00}, 0xc060421e00, 0xc071bdb968) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e55a240}, 0xc048eb4fc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc077aee000, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc077aee000, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811834 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc03b2684e0}, 0xc00065bab8, 0x1, 0xc035e8d0a8, 0x519bbc, 0xc03c141980, 0xc05a07b3e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc07202a120, {0x63c85b8, 0xc03b2684e0}, 0xc07202a1a0, 0xc035e8d1a8, 0x40, 0x60, 0xffffffffffffffff, 0xc03c141980, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc03b2684e0}, 0xc03c1418c0, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc067b5d440, 0x1d}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc03b2684e0}, 0x6352260, 0x42012650) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc03b2684e0}, {0xc03c1417a0, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc03b2684e0}, 0xc050935a40, 0xc035e8d9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc03b2684e0}, 0x660089, {0xc03c1417a0, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc03b2684e0}, 0xc018e6d700, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03b268450}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b268450}, {0xc035e8f3f8, 0x660089}, 0xc035e8f400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03b268450}, {0x0}, 0xc018e6d500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03b2683c0}, 0xc018e6d500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}, 0xc07b87a200, 0xc08b9ff0e0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}, 0x3, 0xc04c7b3680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc035e8f980}}}, {0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}, 0xc07b87a200, 0xc04c7b3680) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc03b2683c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03b2683c0}, {0x40, 0xc035e8fb70}, 0xc035e8fab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc03b2683c0}, {0x4e9d7a0, 0xc018e6d500}, 0xc07b87a200, 0xc08b9ff0e0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc03b2683c0}, 0xc0cc91afc0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03c141680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03c141680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802573 [runnable]: sync.runtime_SemacquireMutex(0xc078f32690, 0x8f, 0x7fddbfb5ad00) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc078f32538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc078f32560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc05fa38000, {0xffffffffffffff15, 0xc078f326c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc05fa38000) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc05fa38000, {0xc017c79c80, 0x32, 0x74}, {0xc02c578a00, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc093255e98, {0xc017c79c80, 0x32, 0x74}, {0xc02c578a00, 0x32, 0x32}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0932547c0, {0xc02c578a00, 0x32, 0x74}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0932547c0, {0xc017c79c80, 0x32, 0xc02c578a00}, {0xc02c578a00, 0xc0d1a5fa40, 0x31}, 0xf0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc093254500, {0xc02c578a00, 0x32, 0x32}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc0ced93328, {{0xc0d1a5fa40, 0x31, 0xf0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc051530800, {{0xc058713c50, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc029e32800, {0x63c85b8, 0xc04c7e6f30}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc04c7e6f30}, {0x64a6560, 0xc051530800}, {0xc058713c50, 0x20838, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc04c7e6f30}, {0x6466ed8, 0xc0ced93080}, {0xc058713c50, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0369d3170}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc0407b1800, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020f1fc00, {0x63c85b8, 0xc04c7e6f30}, 0xc0407b1800, 0xc054757e00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020f1fc00, {0x63c85b8, 0xc04c7e6f30}, 0xc0407b1800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04c7e6ea0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04c7e6ea0}, {0xc0076b08f0, 0x660089}, 0xc0076b08f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04c7e6ea0}, {0xc00102e4e0}, 0xc0407b1700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04c7e6e40}, 0xc0407b1700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc063220ab0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219801363 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc07a6e5e30}, 0xc06775e100, 0xc07d3cc640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc07a6e5e30}, 0xc06775e100, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104dbcbff6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104dbcbff6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07a6e5da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07a6e5da0}, {0xc03851e7f8, 0x660089}, 0xc03851e800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07a6e5da0}, {0x24}, 0xc06775e000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07a6e5d40}, 0xc06775e000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc00bbebc30, {0x63c85b8, 0xc07a6e5bf0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc017846c00, {0x63c85b8, 0xc07a6e5bf0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc017846c00, {0x63c8510, 0xc078e3efc0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc017846c00, {0x63c8510, 0xc078e3efc0}, {0x63f7db8, 0x9bf6f48}, 0xc038523088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc017846c00, {0x63c85b8, 0xc07b5108a0}, {{0x63f7db8, 0x9bf6f48}, {0xc060f8b27e, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc017846c00, {0x63c85b8, 0xc07b5108a0}, {{0x63f7db8, 0x9bf6f48}, {0xc060f8b27e, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc060f8b27e, 0x6}, 0x0, 0x0}, {0x32c3065e, 0xedb7a834f, 0x0}, {0x32c30d9e, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc017846c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc017846c00, {0x63c8510, 0xc0d055cb40}, 0xc075e71dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc073c92970) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0d055cb40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc073c92970) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219802920 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc07584bf20}, 0xc04a5f7fc8, 0x1, 0x0, 0x1, 0xc044dd1ec0, 0xc071370f00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07584bf20}, 0x120, 0xc0549a6728, 0x44dd1e60, 0x0, 0x660069, 0xc044dd1ec0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc07584bf20}, 0xc044dd1e60, {{{{0x0, 0x0}, {0xc07411e6c0, 0xd1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07584bf20}, 0x6352410, 0x49a42258) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc07584bf20}, {0xc040e04c60, {0x174301102097aece, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc07584bf20}, 0xc023a39ae0, 0xc04c2eeee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07584bf20}, 0x660089, {0xc040e04c60, {0x174301102097aece, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc07584bf20}, 0xc0a9a78300, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102097aece, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102097aece, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07584be90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07584be90}, {0xc01fad48f0, 0x660089}, 0xc01fad48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07584be90}, {0xc00102e4e0}, 0xc0a9a78200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07584be30}, 0xc0a9a78200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07cdb8510}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812276 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016c3b2c0, {0x63c85b8, 0xc07b38d7d0}, 0xc032cce0c0, 0x1, 0xc035c910a8, 0x519bbc, 0xc024ed1da0, 0xc0119f7a40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc07b38d7d0}, 0x120, 0xc035c91230, 0x24ed1d40, 0x0, 0x660069, 0xc024ed1da0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016c3b2c0, {0x63c85b8, 0xc07b38d7d0}, 0xc024ed1d40, {{{{0x0, 0x0}, {0xc0cd8fd8c0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc07b38d7d0}, 0x6352410, 0x17838530) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016ce21e0, {0x63c85b8, 0xc07b38d7d0}, {0xc024ed1c20, {0x174301107e50a6b9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016ce21e0, {0x63c85b8, 0xc07b38d7d0}, 0xc02e903f40, 0xc035c919f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc07b38d7d0}, 0x660089, {0xc024ed1c20, {0x174301107e50a6b9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016d2a000, {0x63c85b8, 0xc07b38d7d0}, 0xc04efb6f00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107e50a6b9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc07b38d740}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b38d740}, {0xc035c933f8, 0x660089}, 0xc035c93400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc07b38d740}, {0xc0015221e0}, 0xc04efb6d00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc07b38d680}, 0xc04efb6d00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}, 0xc07cd51f20, 0xc054532210) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}, 0x3, 0xc038dd4cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc035c93980}}}, {0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}, 0xc07cd51f20, 0xc038dd4cc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc07b38d680}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc07b38d680}, {0x40, 0xc035c93b70}, 0xc035c93ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc07b38d680}, {0x4e9d7a0, 0xc04efb6d00}, 0xc07cd51f20, 0xc054532210) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc07b38d680}, 0xc02332bc20, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09fd4d680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc09fd4d680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219730672 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0205cac00, {0x63c85b8, 0xc05934bf80}, 0xc0517d4800, 0xc048083f40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cac00, {0x63c85b8, 0xc05934bf80}, 0xc0517d4800, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc05934bef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc05934bef0}, {0xc07d69a7f8, 0x660089}, 0xc07d69a800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc05934bef0}, {0xc00102e4e0}, 0xc0517d4700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc05934be90}, 0xc0517d4700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc09ad00370, {0x63c85b8, 0xc05934bd40}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc078942000, {0x63c85b8, 0xc05934bd40}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc078942000, {0x63c8510, 0xc04375fd00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc078942000, {0x63c8510, 0xc04375fd00}, {0x63f7db8, 0x9bf6f48}, 0xc07d69f088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc078942000, {0x63c85b8, 0xc078272630}, {{0x63f7db8, 0x9bf6f48}, {0xc028f3c2cb, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc078942000, {0x63c85b8, 0xc078272630}, {{0x63f7db8, 0x9bf6f48}, {0xc028f3c2cb, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc028f3c2cb, 0x6}, 0x0, 0x0}, {0x105a04ba, 0xedb7a834f, 0x0}, {0x105a0bf6, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc078942000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc078942000, {0x63c8510, 0xc03b276f40}, 0xc058197dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc056184760) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc03b276f40}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc056184760) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219812820 [runnable]: github.com/cockroachdb/cockroach/pkg/sql.(*optTable).Equals(0xc001507b80, {0x63586b0, 0xc04b3d7ce0}) github.com/cockroachdb/cockroach/pkg/sql/opt_catalog.go:1033 +0x2bb github.com/cockroachdb/cockroach/pkg/sql/opt.(*Metadata).CheckDependencies(0xc06bb2f4a0, {0x63c85b8, 0xc04c419c80}, {0x646da58, 0xc03f83af88}) github.com/cockroachdb/cockroach/pkg/sql/opt/metadata.go:301 +0x170 github.com/cockroachdb/cockroach/pkg/sql/opt/memo.(*Memo).IsStale(0xc06bb2f4a0, {0x63c85b8, 0xc04c419c80}, 0xc03f83a9e8, {0x646da58, 0xc03f83af88}) github.com/cockroachdb/cockroach/pkg/sql/opt/memo/memo.go:333 +0x4c7 github.com/cockroachdb/cockroach/pkg/sql.(*planner).prepareUsingOptimizer(0xc03f83a6f0, {0x63c85b8, 0xc04c419c80}) github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:129 +0xb6f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).populatePrepared(0xc03f83a000, {0x63c85b8, 0xc04c419c80}, 0xc065f8b5e8, {0xc065f8b5e8, 0x1, 0x1}, 0xc03f83a6f0, 0xc06f2072b0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_prepare.go:287 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).prepare.func2({0x63c85b8, 0xc04c419c80}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_prepare.go:239 +0x4b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).prepare(0xc03f83a000, {0x63c85b8, 0xc04c419c80}, {{{0x63f8d78, 0xc0413e9270}, {0xc090dcf00f, 0x97}, 0x1, 0x1}, {0xc00cb450e0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_prepare.go:244 +0x347 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).addPreparedStmt(0xc03f83a000, {0x63c85b8, 0xc04c419c80}, {0xc090dcf000, 0x0}, {{{0x63f8d78, 0xc0413e9270}, {0xc090dcf00f, 0x97}, 0x1, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_prepare.go:107 +0x110 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPrepare(0xc03f83a000, {0x63c85b8, 0xc04c419c50}, {{0xc090dcf000, 0xe}, {{0x63f8d78, 0xc0413e9270}, {0xc090dcf00f, 0x97}, 0x1, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_prepare.go:68 +0x478 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03f83a000) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1993 +0x10c6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03f83a000, {0x63c8510, 0xc005a6d040}, 0xc06f207dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0a0eb9d30) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc005a6d040}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc0a0eb9d30) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219809267 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc033815540, {0x63c85b8, 0xc0410d01e0}, 0xc06f94d5d8, 0x1, 0x0, 0x1, 0xc08cb19920, 0xc08cb184e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0410d01e0}, 0x120, 0xc00f566728, 0x8cb198c0, 0x0, 0x660069, 0xc08cb19920, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc033815540, {0x63c85b8, 0xc0410d01e0}, 0xc08cb198c0, {{{{0x0, 0x0}, {0xc02833afc0, 0x9d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0410d01e0}, 0x6352410, 0x1fa2e768) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc05c3e4e60, {0x63c85b8, 0xc0410d01e0}, {0xc06870b9e0, {0x17430110636c17c9, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc05c3e4e60, {0x63c85b8, 0xc0410d01e0}, 0xc079c939a0, 0xc07d93eee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0410d01e0}, 0x660089, {0xc06870b9e0, {0x17430110636c17c9, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc0410d01e0}, 0xc049e83700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110636c17c9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110636c17c9, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0410d0150}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0410d0150}, {0xc0936988f0, 0x660089}, 0xc0936988f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0410d0150}, {0xc00102e4e0}, 0xc049e83600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0410d00f0}, 0xc049e83600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc02fdb7530}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803447 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc00b3d3cc0, {0x63c85b8, 0xc023fdcb70}, 0xc0681f3500, 0x1, 0xc06da790a8, 0x519bbc, 0xc039503380, 0xc037e62960) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc023fdcb70}, 0x120, 0xc06da79230, 0x39503320, 0x0, 0x660069, 0xc039503380, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc00b3d3cc0, {0x63c85b8, 0xc023fdcb70}, 0xc039503320, {{{{0x0, 0x0}, {0xc0664bb8c0, 0x18}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc023fdcb70}, 0x6352410, 0x473dd3b8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0309e8550, {0x63c85b8, 0xc023fdcb70}, {0xc039503200, {0x174301102cc46955, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0309e8550, {0x63c85b8, 0xc023fdcb70}, 0xc04d8186e0, 0xc06da799f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc023fdcb70}, 0x660089, {0xc039503200, {0x174301102cc46955, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00036a800, {0x63c85b8, 0xc023fdcb70}, 0xc00fe98700, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102cc46955, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102cc46955, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc023fdc8d0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023fdc8d0}, {0xc06da7b3f8, 0x660089}, 0xc06da7b400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc023fdc8d0}, {0xc0015221e0}, 0xc00fe98500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc023fdc6c0}, 0xc00fe98500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}, 0xc04622f120, 0xc014ba82b8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}, 0x3, 0xc0419b5a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06da7b980}}}, {0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}, 0xc04622f120, 0xc0419b5a00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc023fdc6c0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc023fdc6c0}, {0x40, 0xc06da7bb70}, 0xc06da7bab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc023fdc6c0}, {0x4e9d7a0, 0xc00fe98500}, 0xc04622f120, 0xc014ba82b8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc023fdc6c0}, 0xc042572b40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03896f7a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc03896f7a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219694919 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc06675ee70, {0x63c85b8, 0xc07a25abd0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc035ce2600, {0x63c85b8, 0xc07a25abd0}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc035ce2600, {0x63c8510, 0xc0b1bf0640}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc035ce2600, {0x63c8510, 0xc0b1bf0640}, {0x63f7db8, 0x9bf6f48}, 0xc0955e3088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc035ce2600, {0x63c85b8, 0xc0871245a0}, {{0x63f7db8, 0x9bf6f48}, {0xc08d251e6b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc035ce2600, {0x63c85b8, 0xc0871245a0}, {{0x63f7db8, 0x9bf6f48}, {0xc08d251e6b, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc08d251e6b, 0x6}, 0x0, 0x0}, {0x1eba7c4d, 0xedb7a8350, 0x0}, {0x1eba8469, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc035ce2600) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc035ce2600, {0x63c8510, 0xc06df239c0}, 0xc086ca9dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a545f10) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc06df239c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc04a545f10) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219803716 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc02efadd10}, 0xc0b52ad098, 0x1, 0xc0727730a8, 0x519bbc, 0xc040340f60, 0xc04c116060) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc02efadd10}, 0x1e0, 0xc072773230, 0x40340f00, 0x0, 0x660069, 0xc040340f60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc02efadd10}, 0xc040340f00, {{{{0x0, 0x0}, {0xc0748c5b00, 0x4a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc02efadd10}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc02efadd10}, {0xc043beb320, {0x174301102fcc5c7a, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc02efadd10}, 0xc052058460, 0xc0727739f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc02efadd10}, 0x660089, {0xc043beb320, {0x174301102fcc5c7a, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc02efadd10}, 0xc06e133e00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102fcc5c7a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102fcc5c7a, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc02efadb90}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02efadb90}, {0xc0727753f8, 0x660089}, 0xc072775400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc02efadb90}, {0x1500009005000010}, 0xc06e133c00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc02efadb00}, 0xc06e133c00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}, 0xc04f1afca0, 0xc09e45f7a0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}, 0x3, 0xc038d03340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc072775980}}}, {0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}, 0xc04f1afca0, 0xc038d03340) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc02efadb00}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc02efadb00}, {0x40, 0xc072775b70}, 0xc072775ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc02efadb00}, {0x4e9d7a0, 0xc06e133c00}, 0xc04f1afca0, 0xc09e45f7a0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc02efadb00}, 0xc0bb59e120, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc043beb200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc043beb200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219802547 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc05b267000, {0x63c85b8, 0xc04e4bdcb0}, 0xc01247e900, 0xc0627cdc20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc05b267000, {0x63c85b8, 0xc04e4bdcb0}, 0xc01247e900, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011021df5e1f, 0x0, 0x0}, 0xc015f42770, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011021df5e1f, 0x0, 0x0}, 0xc015f42770, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e4bdc20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e4bdc20}, {0xc063b3d3f8, 0x660089}, 0xc063b3d400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e4bdc20}, {0x7fe4293c8e50}, 0xc0cd6e7f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e4bdb90}, 0xc0cd6e7f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}, 0xc043f00ae0, 0xc0a6a83428) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}, 0x3, 0xc06b8f3e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc063b3d980}}}, {0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}, 0xc043f00ae0, 0xc06b8f3e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e4bdb90}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e4bdb90}, {0x40, 0xc063b3db70}, 0xc063b3dab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}, 0x4beac80, 0x7fe457d81f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e4bdb90}, {0x4e9d7a0, 0xc0cd6e7f00}, 0xc043f00ae0, 0xc0a6a83428) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e4bdb90}, 0xc03e503860, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03854e900, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc03854e900, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810333 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc04ceda870}, 0xc03a3887e0, 0x1, 0xc029ec50a8, 0x519bbc, 0xc025b816e0, 0xc0845102a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc04ceda870}, 0x120, 0xc029ec5230, 0x25b81680, 0x0, 0x660069, 0xc025b816e0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc04ceda870}, 0xc025b81680, {{{{0x0, 0x0}, {0xc0399ecd80, 0x26}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc04ceda870}, 0x6352410, 0x3594a630) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc04ceda870}, {0xc025b81440, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc04ceda870}, 0xc031a2ec80, 0xc029ec59f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc04ceda870}, 0x660089, {0xc025b81440, {0x17430110675268c2, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc04ceda870}, 0xc08a0b3000, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110675268c2, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04ceda7e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ceda7e0}, {0xc029ec73f8, 0x660089}, 0xc029ec7400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04ceda7e0}, {0xc0015221e0}, 0xc08a0b2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04ceda750}, 0xc08a0b2e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}, 0xc090dc0700, 0xc029991320) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}, 0x3, 0xc0301a4080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc029ec7980}}}, {0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}, 0xc090dc0700, 0xc0301a4080) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04ceda750}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04ceda750}, {0x40, 0xc029ec7b70}, 0xc029ec7ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}, 0x4beac80, 0x7fe457d7fa01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04ceda750}, {0x4e9d7a0, 0xc08a0b2e00}, 0xc090dc0700, 0xc029991320) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04ceda750}, 0xc04fccde60, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc025b81200, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc025b81200, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219770328 [runnable]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0140ba360) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:324 +0x85 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:339 google.golang.org/grpc.(*csAttempt).recvMsg(0xc049606160, {0x4d83cc0, 0xc06e8f9680}, 0x6367fb0) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:969 +0xbb google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xd8) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:834 +0x25 google.golang.org/grpc.(*clientStream).withRetry(0xc00b93fe60, 0xc08ef342b0, 0xc08ef34280) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:692 +0xd3 google.golang.org/grpc.(*clientStream).RecvMsg(0xc00b93fe60, {0x4d83cc0, 0xc06e8f9680}) google.golang.org/grpc/external/org_golang_google_grpc/stream.go:833 +0x11f google.golang.org/grpc.invoke({0x63c85b8, 0xc074ce14d0}, {0x4f6406d, 0x2}, {0x4e9d7a0, 0xc070375800}, {0x4d83cc0, 0xc06e8f9680}, 0x2, {0xc012856840, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7 github.com/cockroachdb/cockroach/pkg/util/tracing.ClientInterceptor.func2({0x63c85b8, 0xc074ce14d0}, {0x4f6406d, 0x7fe457d81688}, {0x4e9d7a0, 0xc070375800}, {0x4d83cc0, 0xc06e8f9680}, 0x45339b98f2263ca7, 0x51cbf18, ...) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:300 +0x339 google.golang.org/grpc.(*ClientConn).Invoke(0x21bb180121d5463c, {0x63c85b8, 0xc074ce14d0}, {0x4f6406d, 0x0}, {0x4e9d7a0, 0xc070375800}, {0x4d83cc0, 0xc06e8f9680}, {0x0, ...}) google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc0204d2df8, {0x63c85b8, 0xc074ce14d0}, 0x0, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9245 +0xc9 github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x646abf0, 0xc0204d2df8}}, {0x63c85b8, 0xc074ce14d0}, 0xc070375800, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:332 +0x186 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1c, 0x1c, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:129 +0x725 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/sql/row.makeKVBatchFetcherDefaultSendFunc.func1({_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, 0x0}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:181 +0x69 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).fetch(0xc03ceb52c0, {0x63c85b8, 0xc062057590}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:347 +0x2e5 github.com/cockroachdb/cockroach/pkg/sql/row.(*txnKVFetcher).nextBatch(0xc03ceb52c0, {0x63c85b8, 0xc062057590}) github.com/cockroachdb/cockroach/pkg/sql/row/kv_batch_fetcher.go:531 +0xa45 github.com/cockroachdb/cockroach/pkg/sql/row.(*KVFetcher).NextKV(0xc06b8c19a0, {0x63c85b8, 0xc062057590}, 0x0) github.com/cockroachdb/cockroach/pkg/sql/row/kv_fetcher.go:209 +0xeb github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*cFetcher).NextBatch(0xc08c2a4e00, {0x63c85b8, 0xc062057590}) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/cfetcher.go:690 +0xc6 github.com/cockroachdb/cockroach/pkg/sql/colfetcher.(*ColIndexJoin).Next(0xc00dcfcf20) github.com/cockroachdb/cockroach/pkg/sql/colfetcher/index_join.go:264 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*CancelChecker).Next(0xc03847ea00) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/cancel_checker.go:59 +0x30 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc01d3f7800) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colexec.getNext_false(0xc00a42e500) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/hash_aggregator.eg.go:540 +0xef github.com/cockroachdb/cockroach/pkg/sql/colexec.(*hashAggregator).Next(0x0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/hash_aggregator.eg.go:82 +0x36 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1() github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:198 +0x2f github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x779f4f32604f94e0) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc06f803440) github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:196 +0x68 github.com/cockroachdb/cockroach/pkg/sql/colexec.spool_false(0xc05a34d6c0) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/sorttopk.eg.go:310 +0x3e github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).spool(0xc083606540) github.com/cockroachdb/cockroach/bazel-out/k8-opt/bin/pkg/sql/colexec/sorttopk.eg.go:67 +0x2c github.com/cockroachdb/cockroach/pkg/sql/colexec.(*topKSorter).Next(0xc05a34d6c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/sorttopk.go:165 +0x65 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1() github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:198 +0x2f github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x46afe6) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc06f803560) github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:196 +0x68 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc09ff9c960) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:266 +0x73 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:291 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x7fe457d803c8) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc09ff9c960) github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:297 +0x4c github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc03e19e800) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:747 +0x22d github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc03e19e800) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:666 +0x70 github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).readInput(0xc033c68000) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:747 +0x22d github.com/cockroachdb/cockroach/pkg/sql/rowexec.(*joinReader).Next(0xc033c68000) github.com/cockroachdb/cockroach/pkg/sql/rowexec/joinreader.go:666 +0x70 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc0837a6000) github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:210 +0x130 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*allSpooler).spool(0xc046d3a0c0) github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:136 +0x37 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*sortOp).Next(0xc0cc8df900) github.com/cockroachdb/cockroach/pkg/sql/colexec/sort.go:280 +0x7a github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next.func1() github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:198 +0x2f github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x0) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colexec.(*diskSpillerBase).Next(0xc06f8037a0) github.com/cockroachdb/cockroach/pkg/sql/colexec/disk_spiller.go:196 +0x68 github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc046d3a1e0) github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:125 +0x3f github.com/cockroachdb/cockroach/pkg/sql/colexecop.(*noopOperator).Next(0x19be5e0) github.com/cockroachdb/cockroach/pkg/sql/colexecop/operator.go:387 +0x22 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).nextAdapter(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:240 github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xc09626b800) github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).next(...) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:244 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*BatchFlowCoordinator).Run(0xc040b344d0, {0x63c8510, 0xc03847e980}) github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:287 +0x328 github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0xc07414bc70, {0x63c8510, 0xc03847e980}, 0xc00c937130) github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:316 +0x1d9 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc00c689cc0, {0x63c8510, 0xc06051b1c0}, 0xc08c4f41c0, 0xc00c937130, 0xc07b6bac80, 0xc0625f0a80, 0xc0a14f15e8, 0x0) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:597 +0xcb6 github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x63c8510, {0x63c8510, 0xc06051b1c0}, 0xc0a14f12f0, 0xc08c4f41c0, 0xc07988d140, {{0x63ca960, 0xc0502ba000}, 0x0}, 0xc0625f0a80) github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1445 +0x1b2 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc0a14f0c00, {0x63c8510, 0xc06051b1c0}, 0xc0a14f12f0, 0xc06051b1c0, {0x7fe42b790cf0, 0xc07988d140}, 0x0, 0xc01d3f66b8) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1555 +0x613 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc0a14f0c00, {0x63c8510, 0xc06051b1c0}, 0xc0a14f12f0, {0x7fe42b790cf0, 0xc07988d140}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1212 +0xa2f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc0a14f0c00, {0x63c85b8, 0xc02707a4e0}, {{0x63f8d78, 0xc06235b040}, {0xc03af3a63a, 0x189}, 0x2, 0x2}, 0xc0171cf200, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:747 +0x240b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc0a14f0c00, {0x63c85b8, 0xc02707a4e0}, {{0x63f8d78, 0xc06235b040}, {0xc03af3a63a, 0x189}, 0x2, 0x2}, 0xc0171cf200, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc0a14f0c00, {0x63c85b8, 0xc02707a4e0}, {0xc0171cf200, {0xc0adac6f20, 0x2, 0x2}, {0xc0d5995f80, 0x14, 0x14}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:267 +0x16e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0xc03af3a84e, 0x0}, 0x0, {0x2cb3df64, 0xedb7a8350, 0x0}, 0x1}, 0xc0a14f0c00, 0xc08ef3b980, 0xc08ef3b970, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1969 +0x552 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc0a14f0c00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1971 +0x41b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc0a14f0c00, {0x63c8510, 0xc02341e340}, 0xc0526c7dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc048b6bce0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc02341e340}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc048b6bce0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219794932 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031bac800, {0x63c85b8, 0xc04aa59770}, 0xc04f057600, 0xc0b68cb360) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc04aa59770}, 0xc04f057600, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105bff062f, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04aa596e0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04aa596e0}, {0xc07dc4e7f8, 0x660089}, 0xc07dc4e800) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04aa596e0}, {0xc00102e4e0}, 0xc04f057500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04aa59680}, 0xc04f057500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1006 +0x7df github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc05b9df600, {0x63c85b8, 0xc04aa59530}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc04c29ec00, {0x63c85b8, 0xc04aa59530}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc04c29ec00, {0x63c8510, 0xc011aba0c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction(0xc04c29ec00, {0x63c8510, 0xc011aba0c0}, {0x63f7db8, 0x9bf6f48}, 0xc07dc53088) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:950 +0x8b github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc04c29ec00, {0x63c85b8, 0xc0517e5a70}, {{0x63f7db8, 0x9bf6f48}, {0xc0780b325e, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:582 +0x1cbe github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc04c29ec00, {0x63c85b8, 0xc0517e5a70}, {{0x63f7db8, 0x9bf6f48}, {0xc0780b325e, 0x6}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f7db8, 0x9bf6f48}, {0xc0780b325e, 0x6}, 0x0, 0x0}, {0x5ac7eec, 0xedb7a8350, 0x0}, {0x5ac8590, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc04c29ec00) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc04c29ec00, {0x63c8510, 0xc0383219c0}, 0xc0768eddd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc017bd5950) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0383219c0}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc017bd5950) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219801418 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0743bf410}, 0xc030342d98, 0x1, 0xc0121210a8, 0x519bbc, 0xc037336840, 0xc0ace0d5c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0743bf410}, 0x120, 0xc012121230, 0x373367e0, 0x0, 0x660069, 0xc037336840, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0743bf410}, 0xc0373367e0, {{{{0x0, 0x0}, {0xc0748c4900, 0x31}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0743bf410}, 0x6352410, 0x11938) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0743bf410}, {0xc0373366c0, {0x1743011018bca093, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0743bf410}, 0xc0c5e15720, 0xc0121219f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0743bf410}, 0x660089, {0xc0373366c0, {0x1743011018bca093, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0743bf410}, 0xc04947e100, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011018bca093, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011018bca093, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0743bf380}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0743bf380}, {0xc0121233f8, 0x660089}, 0xc012123400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0743bf380}, {0x7fe3e0eed8f0}, 0xc097299f00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0743bf2f0}, 0xc097299f00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}, 0xc076a265e0, 0xc0388e9788) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}, 0x3, 0xc0b1db9a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc012123980}}}, {0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}, 0xc076a265e0, 0xc0b1db9a80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0743bf2f0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0743bf2f0}, {0x40, 0xc012123b70}, 0xc012123ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}, 0x4beac80, 0x7fe457d7ff01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0743bf2f0}, {0x4e9d7a0, 0xc097299f00}, 0xc076a265e0, 0xc0388e9788) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0743bf2f0}, 0xc020d16d80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0373365a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0373365a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805975 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02fe8ea00, {0x63c85b8, 0xc0239c3b60}, 0xc03dc65620, 0x1, 0xc001c070a8, 0x519bbc, 0xc01cc1f320, 0xc05c5b6ba0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0239c3b60}, 0x1e0, 0xc001c07230, 0x1cc1f2c0, 0x0, 0x660069, 0xc01cc1f320, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02fe8ea00, {0x63c85b8, 0xc0239c3b60}, 0xc01cc1f2c0, {{{{0x0, 0x0}, {0xc09ce318c0, 0x59}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0239c3b60}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02fea2280, {0x63c85b8, 0xc0239c3b60}, {0xc05c2b4360, {0x1743011049532d87, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02fea2280, {0x63c85b8, 0xc0239c3b60}, 0xc071dacc80, 0xc001c079f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0239c3b60}, 0x660089, {0xc05c2b4360, {0x1743011049532d87, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fea7000, {0x63c85b8, 0xc0239c3b60}, 0xc0715df900, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011049532d87, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011049532d87, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0239c3ad0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0239c3ad0}, {0xc001c093f8, 0x660089}, 0xc001c09400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0239c3ad0}, {0x40e8faff2ddd4c3a}, 0xc0715df500) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0239c3a10}, 0xc0715df500) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}, 0xc055efe080, 0xc07c7855d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}, 0x3, 0xc07b830e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc001c09980}}}, {0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}, 0xc055efe080, 0xc07b830e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0239c3a10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0239c3a10}, {0x40, 0xc001c09b70}, 0xc001c09ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}, 0x4beac80, 0x7fe457d93301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0239c3a10}, {0x4e9d7a0, 0xc0715df500}, 0xc055efe080, 0xc07c7855d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0239c3a10}, 0xc048e50480, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05c2b4240, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc05c2b4240, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219681164 [IO wait]: internal/poll.runtime_pollWait(0x7fdb124bc9a0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc05a5fa300, 0xc037a72800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc05a5fa300, {0xc037a72800, 0x739, 0x739}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc05a5fa300, {0xc037a72800, 0x739, 0xc00fbd5180}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc052f31820, {0xc037a72800, 0x7eda79, 0xc037a72805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0377f1d40, {0xc037a72800, 0x0, 0x739}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc03c545e60, {0xc037a72800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc0d121f3f8, {0x631b900, 0xc03c545e60}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc0d121f180, {0x7fe42ab59310, 0xc0377f1d40}, 0x739) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc0d121f180, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc0d121f180, {0xc0035bf000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc01e068978, {0xc0035bf000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc0c9a58088) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc0c9a58088) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc0c9a58660, {0x634aac0, 0xc0c9a58088}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc0c9a58000, 0xc08c6a8ac0, 0xc07de9542b, 0xc07de9542d, {0x635bec0, 0xc00fe63680}, 0xc000d6ae00, 0xc0572de714, 0xc07de95429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc0c9a58000, {0x63c85b8, 0xc061a9b3e0}, 0xc08c6a8a90, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc06216d020, 0xb}}, 0x0, 0xc061a9b380, 0xc061a9b3b0, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc061a9b3e0}, {0x643d810, 0xc0377f1d40}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc01ba0cf30}, {0x643d810, 0xc0377f1d40}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc01ba0cf30}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806244 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e293680, {0x63c85b8, 0xc06ff8d8c0}, 0xc04bf522d0, 0x1, 0xc006fd30a8, 0x519bbc, 0xc047233020, 0xc007f455c0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc06ff8d8c0}, 0x120, 0xc006fd3230, 0x47232fc0, 0x0, 0x660069, 0xc047233020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e293680, {0x63c85b8, 0xc06ff8d8c0}, 0xc047232fc0, {{{{0x0, 0x0}, {0xc08f9d2d80, 0x22}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc06ff8d8c0}, 0x6352410, 0x2016da00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02ce4bf40, {0x63c85b8, 0xc06ff8d8c0}, {0xc047232ea0, {0x174301104c6e8840, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02ce4bf40, {0x63c85b8, 0xc06ff8d8c0}, 0xc0852f1b80, 0xc006fd39f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc06ff8d8c0}, 0x660089, {0xc047232ea0, {0x174301104c6e8840, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e2d8000, {0x63c85b8, 0xc06ff8d8c0}, 0xc08517f600, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301104c6e8840, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301104c6e8840, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc06ff8d800}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ff8d800}, {0xc006fd53f8, 0x660089}, 0xc006fd5400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc06ff8d800}, {0xc0015221e0}, 0xc08517f400) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc06ff8d770}, 0xc08517f400) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}, 0xc07c5b0500, 0xc08ba3f908) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}, 0x3, 0xc06e488600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc006fd5980}}}, {0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}, 0xc07c5b0500, 0xc06e488600) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc06ff8d770}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc06ff8d770}, {0x40, 0xc006fd5b70}, 0xc006fd5ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}, 0x4beac80, 0x7fe457d81101) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc06ff8d770}, {0x4e9d7a0, 0xc08517f400}, 0xc07c5b0500, 0xc08ba3f908) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc06ff8d770}, 0xc04da75ce0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc047232d80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc06cd01860}, 0xc047232d80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219810994 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc09860c2d0, {0x63c85b8, 0xc061cbb0e0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:349 +0x18d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000d3ab40, {0x63c85b8, 0xc061cbb0e0}, {0x4f67ad2, 0x10beec0}, 0xc0387f8fb8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0xdd github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).startHeartbeatLoopLocked.func1() github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:304 +0xf0 created by time.goFunc GOROOT/src/time/sleep.go:180 +0x31 goroutine 219674253 [IO wait]: internal/poll.runtime_pollWait(0x7fdbafeea1e0, 0x72) GOROOT/src/runtime/netpoll.go:303 +0x85 internal/poll.(*pollDesc).wait(0xc04f772f80, 0xc00e958800, 0x0) GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) GOROOT/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc04f772f80, {0xc00e958800, 0x735, 0x735}) GOROOT/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc04f772f80, {0xc00e958800, 0x735, 0xc084fa0000}) GOROOT/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc02923be70, {0xc00e958800, 0x7eda79, 0xc00e958805}) GOROOT/src/net/net.go:183 +0x45 github.com/cockroachdb/cmux.(*MuxConn).Read(0xc047215c80, {0xc00e958800, 0x2, 0x735}) github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 crypto/tls.(*atLeastReader).Read(0xc05bebf698, {0xc00e958800, 0x0, 0x45dbad}) GOROOT/src/crypto/tls/conn.go:778 +0x3d bytes.(*Buffer).ReadFrom(0xc051be3778, {0x631b900, 0xc05bebf698}) GOROOT/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc051be3500, {0x7fe42ab59310, 0xc047215c80}, 0x735) GOROOT/src/crypto/tls/conn.go:800 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc051be3500, 0x0) GOROOT/src/crypto/tls/conn.go:607 +0x112 crypto/tls.(*Conn).readRecord(...) GOROOT/src/crypto/tls/conn.go:575 crypto/tls.(*Conn).Read(0xc051be3500, {0xc038fb6000, 0x1000, 0x0}) GOROOT/src/crypto/tls/conn.go:1278 +0x16f github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc056d7e0c0, {0xc038fb6000, 0x1000, 0x1000}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:1838 +0x116 bufio.(*Reader).fill(0xc087b1f588) GOROOT/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).ReadByte(0xc087b1f588) GOROOT/src/bufio/bufio.go:253 +0x2c github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc087b1fb60, {0x634aac0, 0xc087b1f588}) github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:205 +0x30 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func5(0xc087b1f500, 0xc061a4a290, 0xc0704d942b, 0xc0704d942d, {0x635bec0, 0xc04d3c1dd0}, 0xc000d6ae00, 0xc0b711dda8, 0xc0704d9429) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:400 +0xa5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc087b1f500, {0x63c85b8, 0xc0ce4922a0}, 0xc061a4a280, 0xc000d6ae00, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:540 +0xba5 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveConn(0xc00c689e00, {0x63c85b8, _}, {_, _}, {{{0xc093d35020, 0x8}}, 0x0, 0xc0ce492240, 0xc0ce492270, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:173 +0x2db github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn(0xc00c689e00, {0x63c85b8, 0xc0ce4922a0}, {0x643d810, 0xc047215c80}, 0x58) github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:874 +0xdc6 github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).startServeSQL.func1.1({0x63c85b8, 0xc0280dd080}, {0x643d810, 0xc047215c80}) github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1373 +0xc8 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1({0x63c85b8, 0xc0280dd080}) github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:162 +0x150 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219812526 [runnable]: github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807907 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc01dbf32c0, {0x63c85b8, 0xc034fd0d50}, 0xc04a225f80, 0x1, 0x0, 0x1, 0xc073a6bb60, 0xc033691020) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc034fd0d50}, 0x120, 0xc00e730728, 0x73a6bb00, 0x0, 0x660069, 0xc073a6bb60, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc01dbf32c0, {0x63c85b8, 0xc034fd0d50}, 0xc073a6bb00, {{{{0x0, 0x0}, {0xc01f6ea900, 0x29}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc034fd0d50}, 0x6352410, 0x32430a00) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc01dc14500, {0x63c85b8, 0xc034fd0d50}, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc01dc14500, {0x63c85b8, 0xc034fd0d50}, 0xc07639ed20, 0xc070134ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc034fd0d50}, 0x660089, {0xc073ca86c0, {0x1743011043170678, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01dd05000, {0x63c85b8, 0xc034fd0d50}, 0xc07c29be00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011043170678, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc034fd0c30}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc034fd0c30}, {0xc002fec8f0, 0x660089}, 0xc002fec8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc034fd0c30}, {0xc00102e4e0}, 0xc07c29bb00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc034fd0ba0}, 0xc07c29bb00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc054707d70}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219792246 [select]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc0199ccb00, {0x63c8580, 0xc02e207560}, 0xc04fe66cb0, {0x646cc48, 0xc02e207680}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:251 +0x16d github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0078dea50, {0x63c8580, 0xc02e207560}, 0xedb7a834e, 0xc04cc93440) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:743 +0x3e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x63c8580, 0xc02e207560}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:786 +0x38 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc077311c50}, {0x4f1055b, 0x12}, 0x4a817c800, 0xc04fe66ef8) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x63c85b8, 0xc077311c50}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:784 +0xac github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219803461 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc031184c00, {0x63c85b8, 0xc08291daa0}, 0xc0571e8f00, 0xc054f819a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031184c00, {0x63c85b8, 0xc08291daa0}, 0xc0571e8f00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301102d05a52c, 0x0, 0x0}, 0xc03b0d7900, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301102d05a52c, 0x0, 0x0}, 0xc03b0d7900, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08291da10}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08291da10}, {0xc076589918, 0x660089}, 0xc076589920) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08291da10}, {0xc00102e4e0}, 0xc0571e8e00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08291d9b0}, 0xc0571e8e00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1206 +0x426 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:832 +0x645 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/db.go:222 +0xd8 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1({0x63c8580, 0xc06ba85da0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:278 +0xe8 github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x63c85b8, 0xc08291d920}, {0xc00dfd36b0, 0x24}, 0x4a7dbe371, 0xc027908d40) github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xb4 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.2({0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:286 +0x77 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1({0x63c85b8, 0xc08291d920}) github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:309 +0x218 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219615565 [chan receive]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1010 +0x828 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:830 +0x59b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x269 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x118 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:202 +0x366 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:242 +0x264 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:177 +0x316 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:290 +0x286 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x9d github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:232 +0x4e8 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:532 +0x585 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/db.go:985 +0x14d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:1124 +0x21d github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc08fdd51e0, {0x63c85b8, 0xc017083e60}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:706 +0x1d7 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc03ac8b900, {0x63c85b8, 0xc017083e60}) github.com/cockroachdb/cockroach/pkg/kv/txn.go:741 +0x6a github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc03ac8b900, {0x63c8510, 0xc070419e00}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1022 +0x172 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execRelease(0xc03ac8b900, {0x63c8510, 0xc070419e00}, 0xc02e2addd0, {0x7fe42b790cf0, 0xc0742ae6c0}) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_savepoints.go:148 +0x2a5 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc03ac8b900, {0x63c85b8, 0xc02fdb6270}, {{0x63f88f8, 0xc02e2addd0}, {0xc073ae6cc8, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:594 +0x1bca github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc03ac8b900, {0x63c85b8, 0xc02fdb6270}, {{0x63f88f8, 0xc02e2addd0}, {0xc073ae6cc8, 0x23}, 0x0, 0x0}, 0x0, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:179 +0x59e github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1({{{0x63f88f8, 0xc02e2addd0}, {0xc073ae6cc8, 0x23}, 0x0, 0x0}, {0xdad4b64, 0xedb7a8350, 0x0}, {0xdad52a9, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1891 +0x2f6 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc03ac8b900) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1895 +0xb08 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc03ac8b900, {0x63c8510, 0xc0b7add580}, 0xc0282f5dd0, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc016e0d8f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1817 +0x278 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0x63c8510, {0x63c8510, 0xc0b7add580}, {0x3}, {0x5400, 0x15000, 0xc00cc46cc0, 0x0, 0x0}, 0xc016e0d8f0) github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:763 +0xf7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1() github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:728 +0x415 created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:639 +0x273 goroutine 219811925 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0214d8000, {0x63c85b8, 0xc024a55f20}, 0xc04479afa8, 0x1, 0x0, 0x1, 0xc06fafc3c0, 0xc0b9cafa40) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc024a55f20}, 0x120, 0xc033886728, 0x6fafc360, 0x0, 0x660069, 0xc06fafc3c0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0214d8000, {0x63c85b8, 0xc024a55f20}, 0xc06fafc360, {{{{0x0, 0x0}, {0xc03d6f3d40, 0xd}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc024a55f20}, 0x6352410, 0x3c90c088) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02135bb30, {0x63c85b8, 0xc024a55f20}, {0xc06fafc000, {0x174301107747cda6, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02135bb30, {0x63c85b8, 0xc024a55f20}, 0xc0b7e774a0, 0xc07d278ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc024a55f20}, 0x660089, {0xc06fafc000, {0x174301107747cda6, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0214da000, {0x63c85b8, 0xc024a55f20}, 0xc010bc4c00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107747cda6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107747cda6, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc024a55da0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc024a55da0}, {0xc0936368f0, 0x660089}, 0xc0936368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc024a55da0}, {0xc00102e4e0}, 0xc010bc4900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc024a55d40}, 0xc010bc4900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc024a55a40}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219810817 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0252e1a40, {0x63c85b8, 0xc046387320}, 0xc038bf32f0, 0x1, 0xc0273930a8, 0x519bbc, 0xc01cd6cba0, 0xc0681f7b60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc046387320}, 0x120, 0xc027393230, 0x1cd6cb40, 0x0, 0x660069, 0xc01cd6cba0, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0252e1a40, {0x63c85b8, 0xc046387320}, 0xc01cd6cb40, {{{{0x0, 0x0}, {0xc03eb87200, 0x2d}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc046387320}, 0x6352410, 0x1978a878) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02528f4f0, {0x63c85b8, 0xc046387320}, {0xc01cd6ca20, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02528f4f0, {0x63c85b8, 0xc046387320}, 0xc03e556820, 0xc0273939f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc046387320}, 0x660089, {0xc01cd6ca20, {0x174301106bba5a8c, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02532b400, {0x63c85b8, 0xc046387320}, 0xc072b39800, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301106bba5a8c, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046387290}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046387290}, {0xc0273953f8, 0x660089}, 0xc027395400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046387290}, {0x3}, 0xc072b39600) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046387200}, 0xc072b39600) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}, 0xc01ed82780, 0xc03a3882e8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}, 0x3, 0xc047938e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc027395980}}}, {0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}, 0xc01ed82780, 0xc047938e80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc046387200}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046387200}, {0x40, 0xc027395b70}, 0xc027395ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc046387200}, {0x4e9d7a0, 0xc072b39600}, 0xc01ed82780, 0xc03a3882e8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc046387200}, 0xc01cd8cea0, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03bf5cb40, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc03bf5cb40, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219793188 [runnable]: sync.runtime_SemacquireMutex(0x32, 0x44, 0xc07ac9a548) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074ab60, 0xc07ac9a5b0, 0xc0857fd8c0, 0xc0857fd8c0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x0, 0x0, 0x0, 0x0) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc075cf2000, {0x116e025, 0xc0be1611f0}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*Reader).readFilter(...) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2250 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE(0xc04990f500, {0xc0505b9ab0, 0x2b, 0x70}, {0xc084085da0, 0x2b, 0x2b}, 0x1, 0x11) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:633 +0xe9 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).SeekPrefixGE(0xc0be161088, {0xc0505b9ab0, 0x1, 0x2b}, {0xc084085da0, 0x0, 0x0}, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:616 +0x2b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0be161088, {0xc0505b9ab0, 0x2b, 0x70}, {0xc084085da0, 0x2b, 0x2b}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0be1607c0, {0xc084085da0, 0x2b, 0x70}, 0x2b, 0x2b) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0be1607c0, {0xc0505b9ab0, 0x2b, 0xc084085da0}, {0xc084085da0, 0x2a, 0x2a}, 0x0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0be160500, {0xc084085da0, 0x2b, 0x2b}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekGE(0xc090b86108, {{0xc04b0665d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:288 +0xcb github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc046381700, {{0xc04b0665d0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:310 +0x10f github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc04ea44000, {0x63c85b8, 0xc0448f7f80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc0448f7f80}, {0x64a6560, 0xc046381700}, {0xc04b0665d0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc0448f7f80}, {0x6466ed8, 0xc090b86100}, {0xc04b0665d0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc02329bc80}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06cdfaf00, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01fa86400, {0x63c85b8, 0xc0448f7f80}, 0xc06cdfaf00, 0xc018952500) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01fa86400, {0x63c85b8, 0xc0448f7f80}, 0xc06cdfaf00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0448f7ef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0448f7ef0}, {0xc0936368f0, 0x660089}, 0xc0936368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0448f7ef0}, {0xc00102e4e0}, 0xc06cdfae00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0448f7e90}, 0xc06cdfae00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x5, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc01b9447b0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219808187 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0308908c0, {0x63c85b8, 0xc0677ee720}, 0xc08b9ff218, 0x1, 0xc0060cd0a8, 0x519bbc, 0xc04ec64600, 0xc066d19320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x1e0, {0x63c85b8, 0xc0677ee720}, 0x1e0, 0xc0060cd230, 0x4ec645a0, 0x0, 0x660069, 0xc04ec64600, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0308908c0, {0x63c85b8, 0xc0677ee720}, 0xc04ec645a0, {{{{0x0, 0x0}, {0xc0ad638900, 0x4a}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x0, {0x63c85b8, 0xc0677ee720}, 0x0, 0x0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0301af310, {0x63c85b8, 0xc0677ee720}, {0xc0519865a0, {0x174301105b266044, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0301af310, {0x63c85b8, 0xc0677ee720}, 0xc02c9ffb80, 0xc0060cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0677ee720}, 0x660089, {0xc0519865a0, {0x174301105b266044, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc031bac800, {0x63c85b8, 0xc0677ee720}, 0xc057136400, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301105b266044, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301105b266044, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc0677ee630}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0677ee630}, {0xc0060cf3f8, 0x660089}, 0xc0060cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc0677ee630}, {0x7fe42935ddb8}, 0xc057136200) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc0677ee5a0}, 0xc057136200) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}, 0xc0156d14a0, 0xc04309ff68) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}, 0x3, 0xc047e7e700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0060cf980}}}, {0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}, 0xc0156d14a0, 0xc047e7e700) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc0677ee5a0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc0677ee5a0}, {0x40, 0xc0060cfb70}, 0xc0060cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}, 0x4beac80, 0x7fe457d81b01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc0677ee5a0}, {0x4e9d7a0, 0xc057136200}, 0xc0156d14a0, 0xc04309ff68) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc0677ee5a0}, 0xc04dc84d80, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc051986480, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc051986480, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803336 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc02fda2000, {0x63c85b8, 0xc01fb5d050}, 0xc0c5e1ca00, 0xc06477dd60) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02fda2000, {0x63c85b8, 0xc01fb5d050}, 0xc0c5e1ca00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011025e86b69, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011025e86b69, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01fb5cfc0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fb5cfc0}, {0xc06fecb3f8, 0x660089}, 0xc06fecb400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01fb5cfc0}, {0x7fe34bb6fee0}, 0xc0c5e1c700) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01fb5ce10}, 0xc0c5e1c700) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}, 0xc0276bcee0, 0xc02684d308) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}, 0x3, 0xc065e5b7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc06fecb980}}}, {0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}, 0xc0276bcee0, 0xc065e5b7c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01fb5ce10}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01fb5ce10}, {0x40, 0xc06fecbb70}, 0xc06fecbab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}, 0x4beac80, 0x7fe457d82401) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01fb5ce10}, {0x4e9d7a0, 0xc0c5e1c700}, 0xc0276bcee0, 0xc02684d308) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01fb5ce10}, 0xc029def200, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc044be45a0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc044be45a0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219801187 [runnable]: sync.runtime_SemacquireMutex(0xc053e92690, 0x8f, 0x7fdf9c81f9c0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x4, 0xc053e92538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc053e92560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0131c3b00, {0xffffffffffffff15, 0xc053e926c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0131c3b00) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0131c3b00, {0xc0314ae100, 0x32, 0x75}, {0xc02bfc9880, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc0933f4398, {0xc0314ae100, 0x32, 0x75}, {0xc02bfc9880, 0x32, 0x74}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc0933f2cc0, {0xc02bfc9880, 0x74, 0x75}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc0933f2cc0, {0xc0314ae100, 0x74, 0xc02bfc9880}, {0xc02bfc9880, 0xc075eabb90, 0x31}, 0x90) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc0933f2a00, {0xc02bfc9880, 0x32, 0x74}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc098d7f428, {{0xc075eabb90, 0x31, 0x90}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc02122c500, {{0xc04c20c600, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc0777f6800, {0x63c85b8, 0xc046d66ab0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc046d66ab0}, {0x64a6560, 0xc02122c500}, {0xc04c20c600, 0xc00bc1fa40, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc046d66ab0}, {0x6466ed8, 0xc098d7f180}, {0xc04c20c600, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0595794e8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc06358a200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc020c23000, {0x63c85b8, 0xc046d66ab0}, 0xc06358a200, 0xc048bf25a0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc020c23000, {0x63c85b8, 0xc046d66ab0}, 0xc06358a200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc046d66a20}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc046d66a20}, {0xc0562c48f0, 0x660089}, 0xc0562c48f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc046d66a20}, {0xc00102e4e0}, 0xc06358a100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc046d669c0}, 0xc06358a100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc076faf980}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219802689 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc01d21b400, {0x63c85b8, 0xc04e4989c0}, 0xc0890a9d00, 0xc088618000) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:193 +0x8e5 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01d21b400, {0x63c85b8, 0xc04e4989c0}, 0xc0890a9d00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010ffe9cd59d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010ffe9cd59d, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x6, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc04e498930}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e498930}, {0xc0554233f8, 0x660089}, 0xc055423400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc04e498930}, {0x7fe3ba7da478}, 0xc0890a9b00) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc04e498870}, 0xc0890a9b00) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}, 0xc039a9fd20, 0xc07db210f8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}, 0x3, 0xc05f65f4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc055423980}}}, {0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}, 0xc039a9fd20, 0xc05f65f4c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc04e498870}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc04e498870}, {0x40, 0xc055423b70}, 0xc055423ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}, 0x4beac80, 0x7fe457d80801) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc04e498870}, {0x4e9d7a0, 0xc0890a9b00}, 0xc039a9fd20, 0xc07db210f8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc04e498870}, 0xc01bf2c780, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04652d0e0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04652d0e0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219803304 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc0207c4dc0, {0x63c85b8, 0xc08926d2c0}, 0xc02c29d068, 0x1, 0x0, 0x1, 0xc033fcc960, 0xc071ebb380) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc08926d2c0}, 0x120, 0xc07365a728, 0x33fcc900, 0x0, 0x660069, 0xc033fcc960, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc0207c4dc0, {0x63c85b8, 0xc08926d2c0}, 0xc033fcc900, {{{{0x0, 0x0}, {0xc052db17a0, 0x1}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc08926d2c0}, 0x6352410, 0x3ad19fc0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc0207ce550, {0x63c85b8, 0xc08926d2c0}, {0xc0328e4a20, {0x1743011019292be5, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0207ce550, {0x63c85b8, 0xc08926d2c0}, 0xc0389f1900, 0xc076114ee8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc08926d2c0}, 0x660089, {0xc0328e4a20, {0x1743011019292be5, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0205cac00, {0x63c85b8, 0xc08926d2c0}, 0xc040a2aa00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011019292be5, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc08926d230}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc08926d230}, {0xc0936368f0, 0x660089}, 0xc0936368f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc08926d230}, {0xc00102e4e0}, 0xc040a2a900) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc08926d1d0}, 0xc040a2a900) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc07503c390}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219807383 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02e898c80, {0x63c85b8, 0xc053e48e70}, 0xc043eb9038, 0x1, 0xc0064cd0a8, 0x519bbc, 0xc0b9b53b80, 0xc05bce7740) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc00d87e480, {0x63c85b8, 0xc053e48e70}, 0xc0b9b53b80, 0xc0064cd1a8, 0x31, 0x40, 0x0, 0xc0b9b53b80, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02e898c80, {0x63c85b8, 0xc053e48e70}, 0xc0b9b53880, {{{{0x0, 0x0}, {0x0, 0x0}}, {{0xc04e62eb40, 0x1}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc053e48e70}, 0x63521a0, 0x1b929230) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc02e88c820, {0x63c85b8, 0xc053e48e70}, {0xc04087c7e0, {0x17430110077810ff, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc02e88c820, {0x63c85b8, 0xc053e48e70}, 0xc06891adc0, 0xc0064cd9f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc053e48e70}, 0x660089, {0xc04087c7e0, {0x17430110077810ff, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02e8c6000, {0x63c85b8, 0xc053e48e70}, 0xc0983a9400, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110077810ff, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110077810ff, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x4, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc053e48de0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053e48de0}, {0xc0064cf3f8, 0x660089}, 0xc0064cf400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc053e48de0}, {0xc0015221e0}, 0xc0983a9000) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc053e48d20}, 0xc0983a9000) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}, 0xc06d003940, 0xc0297acea0) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}, 0x3, 0xc059362b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc0064cf980}}}, {0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}, 0xc06d003940, 0xc059362b80) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc053e48d20}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc053e48d20}, {0x40, 0xc0064cfb70}, 0xc0064cfab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}, 0x4beac80, 0x7fe457d92f01) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc053e48d20}, {0x4e9d7a0, 0xc0983a9000}, 0xc06d003940, 0xc0297acea0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc053e48d20}, 0xc011e9dd40, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04087c6c0, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc0180c6340}, 0xc04087c6c0, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219805362 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc016b64f00, {0x63c85b8, 0xc01f16acf0}, 0xc0180aab28, 0x1, 0xc076d130a8, 0x519bbc, 0xc058e75020, 0xc059e656e0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc01f16acf0}, 0x120, 0xc076d13230, 0x58e74fc0, 0xc060727040, 0x18, 0xc058e75020, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc016b64f00, {0x63c85b8, 0xc01f16acf0}, 0xc058e74fc0, {{{{0x0, 0x0}, {0xc00e2e6ea0, 0x6}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc01f16acf0}, 0x6352410, 0x3f6a1a20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc016a59040, {0x63c85b8, 0xc01f16acf0}, {0xc058e74ea0, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc016a59040, {0x63c85b8, 0xc01f16acf0}, 0xc043d49720, 0xc076d139f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc01f16acf0}, 0x660089, {0xc058e74ea0, {0x1743011040d25ab8, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc016c4d000, {0x63c85b8, 0xc01f16acf0}, 0xc06702eb00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743011040d25ab8, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc01f16ac60}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01f16ac60}, {0xc076d153f8, 0x660089}, 0xc076d15400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc01f16ac60}, {0x7fe3b8b35428}, 0xc06702e800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc01f16aba0}, 0xc06702e800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}, 0xc065da66e0, 0xc0644d8eb8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}, 0x3, 0xc074900bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc076d15980}}}, {0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}, 0xc065da66e0, 0xc074900bc0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc01f16aba0}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc01f16aba0}, {0x40, 0xc076d15b70}, 0xc076d15ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}, 0x20301d, 0x20301d) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc01f16aba0}, {0x4e9d7a0, 0xc06702e800}, 0xc065da66e0, 0xc0644d8eb8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc01f16aba0}, 0xc0c4129680, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a49cd80, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01979b6c0}, 0xc05a49cd80, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219790160 [runnable]: sync.runtime_SemacquireMutex(0xc0758ec690, 0x8f, 0xc0758ec4a0) GOROOT/src/runtime/sema.go:71 +0x25 sync.(*RWMutex).RLock(...) GOROOT/src/sync/rwmutex.go:63 github.com/cockroachdb/pebble/internal/cache.(*shard).Get(0xc00074a750, 0x11948b8, 0x3, 0xc0758ec538) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:117 +0x4c github.com/cockroachdb/pebble/internal/cache.(*Cache).Get(0x34, 0x2b, 0x34, 0xc0758ec560) github.com/cockroachdb/pebble/internal/cache/external/com_github_cockroachdb_pebble/internal/cache/clockpro.go:742 +0x3c github.com/cockroachdb/pebble/sstable.(*Reader).readBlock(0xc00e0b2580, {0x8019b15, 0x1d4b5}, 0x0, 0x0) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:2292 +0x74 github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).readBlockWithStats(0xc0c42e8900, {0xffffffffffffff15, 0xc0758ec6c0}, 0x1195605) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:398 +0x2f github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex(0xc0c42e8900) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1211 +0x185 github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).SeekPrefixGE(0xc0c42e8900, {0xc02c62cf80, 0x32, 0x73}, {0xc02d339780, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/sstable/external/com_github_cockroachdb_pebble/sstable/reader.go:1382 +0x47b github.com/cockroachdb/pebble.(*levelIter).SeekPrefixGE(0xc09279ed98, {0xc02c62cf80, 0x32, 0x73}, {0xc02d339780, 0x32, 0x77}, 0x1) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/level_iter.go:434 +0x119 github.com/cockroachdb/pebble.(*mergingIter).seekGE(0xc09279d6c0, {0xc02d339780, 0x77, 0x73}, 0x32, 0x32) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:842 +0x10a github.com/cockroachdb/pebble.(*mergingIter).SeekPrefixGE(0xc09279d6c0, {0xc02c62cf80, 0x77, 0xc02d339780}, {0xc02d339780, 0xc0728bbdc0, 0x31}, 0xe0) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/merging_iter.go:911 +0x96 github.com/cockroachdb/pebble.(*Iterator).SeekPrefixGE(0xc09279d400, {0xc02d339780, 0x32, 0x77}) github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/iterator.go:1053 +0x4b8 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).SeekEngineKeyGEWithLimit(0xc04875a828, {{0xc0728bbdc0, 0x31, 0xe0}, {0x0, 0x0, 0x0}}, {0x0, 0x0, 0x0}) github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:330 +0x145 github.com/cockroachdb/cockroach/pkg/storage.(*intentInterleavingIter).SeekGE(0xc02c029c00, {{0xc02971e7e0, 0x2a, 0x2a}, {0x0, 0x0, 0x0}}) github.com/cockroachdb/cockroach/pkg/storage/intent_interleaving_iter.go:337 +0x406 github.com/cockroachdb/cockroach/pkg/storage.(*pebbleMVCCScanner).get(0xc044335000, {0x63c85b8, 0xc03f0d1f80}) github.com/cockroachdb/cockroach/pkg/storage/pebble_mvcc_scanner.go:407 +0x88 github.com/cockroachdb/cockroach/pkg/storage.mvccGet({0x63c85b8, 0xc03f0d1f80}, {0x64a6560, 0xc02c029c00}, {0xc02971e7e0, 0x0, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:769 +0x40a github.com/cockroachdb/cockroach/pkg/storage.MVCCGet({0x63c85b8, 0xc03f0d1f80}, {0x6466ed8, 0xc04875a580}, {0xc02971e7e0, 0x2a, 0x2a}, {0x1743010b884ffd00, 0x0, 0x0}, ...) github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:729 +0x211 github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.Get({_, _}, {_, _}, {{0x64d25c8, 0xc0373638a8}, {{0x1743010b884ffd00, 0x0, 0x0}, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_get.go:55 +0x357 github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand({_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, 0x0}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:495 +0x38a github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch({_, _}, {_, _}, {_, _}, {_, _}, _, 0xc035ba4200, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_evaluate.go:270 +0xbfb github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(_, {_, _}, {_, _}, {_, _}, _, {{0x1743010b884ffd00, 0x0, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:287 +0x385 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc01f61f400, {0x63c85b8, 0xc03f0d1f80}, 0xc035ba4200, 0xc074cfc640) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_read.go:83 +0x3aa github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc01f61f400, {0x63c85b8, 0xc03f0d1f80}, 0xc035ba4200, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:460 +0x3cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x1743010b884ffd00, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc03f0d1ef0}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc03f0d1ef0}, {0xc0ad83a8f0, 0x660089}, 0xc0ad83a8f8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc03f0d1ef0}, {0xc00102e4e0}, 0xc035ba4100) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc03f0d1e90}, 0xc035ba4100) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.Batch(...) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:609 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(_, {_, _}, _, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:209 +0x15e github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:191 +0x12b github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2066 +0x126f github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x0, 0x0, 0x0, ...}, ...}, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1604 +0xce5 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatchAsync.func1({0x63c85b8, 0xc03f0d1bf0}) github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1477 +0x16a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:494 +0x16f created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:483 +0x445 goroutine 219806733 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc024ab4a00, {0x63c85b8, 0xc0903981b0}, 0xc022670cd8, 0x1, 0xc0032210a8, 0x519bbc, 0xc04c0fa7e0, 0xc05f6d0d20) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0xc042835d40, {0x63c85b8, 0xc0903981b0}, 0xc04c0fa840, 0xc0032211a8, 0x66, 0x70, 0x2, 0xc04c0fa7e0, 0x51c47f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc024ab4a00, {0x63c85b8, 0xc0903981b0}, 0xc04c0fa780, {{{{0x0, 0x0}, {0xc0787887e0, 0x2}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:479 +0x379 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x70, {0x63c85b8, 0xc0903981b0}, 0x0, 0x5) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc024947c70, {0x63c85b8, 0xc0903981b0}, {0xc047a23320, {0x17430110509822ac, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc024947c70, {0x63c85b8, 0xc0903981b0}, 0xc067024aa0, 0xc0032219f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0903981b0}, 0x660089, {0xc047a23320, {0x17430110509822ac, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc024ad3c00, {0x63c85b8, 0xc0903981b0}, 0xc097d24c00, 0x51c4320) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x17430110509822ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:156 +0x696 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x17430110509822ac, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc090398120}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090398120}, {0xc0032233f8, 0x660089}, 0xc003223400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc090398120}, {0x3fefffff776812dc}, 0xc097d24800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc090398090}, 0xc097d24800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}, 0xc09097e100, 0xc0226718d8) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}, 0x3, 0xc08cfa28c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc003223980}}}, {0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}, 0xc09097e100, 0xc08cfa28c0) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc090398090}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc090398090}, {0x40, 0xc003223b70}, 0xc003223ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}, 0x4beac80, 0x7fe457d80301) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc090398090}, {0x4e9d7a0, 0xc097d24800}, 0xc09097e100, 0xc0226718d8) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc090398090}, 0xc0337ce660, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0d29f1680, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc01347c1a0}, 0xc0d29f1680, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294 goroutine 219811569 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).waitForSignal(0xc02ff688c0, {0x63c85b8, 0xc0353757a0}, 0xc04fe46d80, 0x1, 0xc0348110a8, 0x519bbc, 0xc0be724f00, 0xc007904a80) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:535 +0x258 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).iterAndWait(0x120, {0x63c85b8, 0xc0353757a0}, 0x120, 0xc034811230, 0xbe724ea0, 0x0, 0x660069, 0xc0be724f00, 0x51c47e8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:517 +0x12e github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).wait(0xc02ff688c0, {0x63c85b8, 0xc0353757a0}, 0xc0be724ea0, {{{{0x0, 0x0}, {0xc0a833cb40, 0x7}}, {{0x0, 0x0}, ...}}}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:467 +0x2d9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch.(*Manager).Acquire(0x16, {0x63c85b8, 0xc0353757a0}, 0x6352410, 0x204d22f8) github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanlatch/manager.go:220 +0x1ad github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*latchManagerImpl).Acquire(0xc030006140, {0x63c85b8, 0xc0353757a0}, {0xc0be724d80, {0x174301107c36161b, 0x0, 0x0}, 0x0, 0x0, 0x0, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/latch_manager.go:28 +0x37 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc030006140, {0x63c85b8, 0xc0353757a0}, 0xc04151cb40, 0xc0348119f0) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:274 +0x550 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0x9bf6f48, {0x63c85b8, 0xc0353757a0}, 0x660089, {0xc0be724d80, {0x174301107c36161b, 0x0, 0x0}, 0x0, 0x0, ...}, ...) github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:222 +0x2cc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc02ff75c00, {0x63c85b8, 0xc0353757a0}, 0xc0827a4a00, 0x51c4318) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:424 +0x374 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(_, {_, _}, {{{0x174301107c36161b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:152 +0x631 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(_, {_, _}, {{{0x174301107c36161b, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_send.go:197 +0x710 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(_, {_, _}, {{{0x0, 0x0, 0x0}, 0x0, {0x1f, 0x1f, 0x2, ...}, ...}, ...}) github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/stores.go:191 +0x1c8 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1({0x63c85b8, 0xc035375710}) github.com/cockroachdb/cockroach/pkg/server/node.go:1006 +0x348 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc035375710}, {0xc0348133f8, 0x660089}, 0xc034813400) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc007e56900, {0x63c85b8, 0xc035375710}, {0x7fe3bddd0450}, 0xc0827a4800) github.com/cockroachdb/cockroach/pkg/server/node.go:989 +0x149 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc007e56900, {0x63c85b8, 0xc035375650}, 0xc0827a4800) github.com/cockroachdb/cockroach/pkg/server/node.go:1058 +0xfc github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9493 +0x78 github.com/cockroachdb/cockroach/pkg/util/tracing.ServerInterceptor.func1({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}, 0xc06b234e80, 0xc04fe47d70) github.com/cockroachdb/cockroach/pkg/util/tracing/grpc_interceptor.go:136 +0x24c google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1117 +0x5b github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func3({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}, 0x3, 0xc0601ffc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:238 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.unaryInterceptor({{{0xc034813980}}}, {0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}, 0xc06b234e80, 0xc0601ffc00) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:72 +0x173 google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1.1({0x63c85b8, 0xc035375650}) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:207 +0x39 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000d3ab40, {0x63c85b8, 0xc035375650}, {0x40, 0xc034813b70}, 0xc034813ab8) github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:344 +0xdd github.com/cockroachdb/cockroach/pkg/rpc.NewServer.func1({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}, 0x4beac80, 0x7fe457d81601) github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:205 +0x9e google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1120 +0x83 google.golang.org/grpc.chainUnaryInterceptors.func1({0x63c85b8, 0xc035375650}, {0x4e9d7a0, 0xc0827a4800}, 0xc06b234e80, 0xc04fe47d70) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1122 +0x12b github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler({0x4e58300, 0xc007e56900}, {0x63c85b8, 0xc035375650}, 0xc04f98e000, 0xc007b6c0a0) github.com/cockroachdb/cockroach/pkg/roachpb/bazel-out/k8-opt/bin/pkg/roachpb/roachpb_go_proto_/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:9495 +0x138 google.golang.org/grpc.(*Server).processUnaryRPC(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0be724c60, 0xc007b7b5f0, 0x91b8a00, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1283 +0xcf2 google.golang.org/grpc.(*Server).handleStream(0xc000f02e00, {0x6446ff8, 0xc019573860}, 0xc0be724c60, 0x0) google.golang.org/grpc/external/org_golang_google_grpc/server.go:1620 +0xa2a google.golang.org/grpc.(*Server).serveStreams.func1.2() google.golang.org/grpc/external/org_golang_google_grpc/server.go:922 +0x98 created by google.golang.org/grpc.(*Server).serveStreams.func1 google.golang.org/grpc/external/org_golang_google_grpc/server.go:920 +0x294